--------------------------------------------------------------------------------
-- | A store for storing and retreiving items
{-# LANGUAGE ExistentialQuantification #-}
{-# LANGUAGE ScopedTypeVariables       #-}
module Hakyll.Core.Store
    ( Store
    , Result (..)
    , toMaybe
    , new
    , set
    , get
    , isMember
    , delete
    , hash
    ) where


--------------------------------------------------------------------------------
import           Control.Monad        (when)
import           Data.Binary          (Binary, decode, encodeFile)
import qualified Data.ByteString.Lazy as BL
import qualified Data.Cache.LRU.IO    as Lru
import qualified Data.Hashable        as DH
import qualified Data.IORef           as IORef
import           Data.List            (intercalate)
import qualified Data.Map             as Map
import           Data.Maybe           (isJust)
import           Data.Typeable        (TypeRep, Typeable, cast, typeOf)
import           System.Directory     (createDirectoryIfMissing, doesFileExist,
                                       removeFile)
import           System.FilePath      ((</>))
import           System.IO            (IOMode (..), hClose, openFile)
import           System.IO.Error      (catchIOError, ioeSetFileName,
                                       ioeSetLocation, modifyIOError)


--------------------------------------------------------------------------------
-- | Simple wrapper type
data Box = forall a. Typeable a => Box a


--------------------------------------------------------------------------------
data Store = Store
    { -- | All items are stored on the filesystem
      Store -> [Char]
storeDirectory  :: FilePath
    , -- | See 'set'
      Store -> IORef (Map [Char] Box)
storeWriteAhead :: IORef.IORef (Map.Map String Box)
      -- | Optionally, items are also kept in-memory
    , Store -> Maybe (AtomicLRU [Char] Box)
storeMap        :: Maybe (Lru.AtomicLRU FilePath Box)
    }


--------------------------------------------------------------------------------
instance Show Store where
    show :: Store -> [Char]
show Store
_ = [Char]
"<Store>"


--------------------------------------------------------------------------------
-- | Result of a store query
data Result a
    = Found a                    -- ^ Found, result
    | NotFound                   -- ^ Not found
    | WrongType TypeRep TypeRep  -- ^ Expected, true type
    deriving (Int -> Result a -> ShowS
[Result a] -> ShowS
Result a -> [Char]
(Int -> Result a -> ShowS)
-> (Result a -> [Char]) -> ([Result a] -> ShowS) -> Show (Result a)
forall a. Show a => Int -> Result a -> ShowS
forall a. Show a => [Result a] -> ShowS
forall a. Show a => Result a -> [Char]
forall a.
(Int -> a -> ShowS) -> (a -> [Char]) -> ([a] -> ShowS) -> Show a
$cshowsPrec :: forall a. Show a => Int -> Result a -> ShowS
showsPrec :: Int -> Result a -> ShowS
$cshow :: forall a. Show a => Result a -> [Char]
show :: Result a -> [Char]
$cshowList :: forall a. Show a => [Result a] -> ShowS
showList :: [Result a] -> ShowS
Show, Result a -> Result a -> Bool
(Result a -> Result a -> Bool)
-> (Result a -> Result a -> Bool) -> Eq (Result a)
forall a. Eq a => Result a -> Result a -> Bool
forall a. (a -> a -> Bool) -> (a -> a -> Bool) -> Eq a
$c== :: forall a. Eq a => Result a -> Result a -> Bool
== :: Result a -> Result a -> Bool
$c/= :: forall a. Eq a => Result a -> Result a -> Bool
/= :: Result a -> Result a -> Bool
Eq)


--------------------------------------------------------------------------------
-- | Convert result to 'Maybe'
toMaybe :: Result a -> Maybe a
toMaybe :: forall a. Result a -> Maybe a
toMaybe (Found a
x) = a -> Maybe a
forall a. a -> Maybe a
Just a
x
toMaybe Result a
_         = Maybe a
forall a. Maybe a
Nothing


--------------------------------------------------------------------------------
-- | Initialize the store
new :: Bool      -- ^ Use in-memory caching
    -> FilePath  -- ^ Directory to use for hard disk storage
    -> IO Store  -- ^ Store
new :: Bool -> [Char] -> IO Store
new Bool
inMemory [Char]
directory = do
    Bool -> [Char] -> IO ()
createDirectoryIfMissing Bool
True [Char]
directory
    writeAhead <- Map [Char] Box -> IO (IORef (Map [Char] Box))
forall a. a -> IO (IORef a)
IORef.newIORef Map [Char] Box
forall k a. Map k a
Map.empty
    ref <- if inMemory then Just <$> Lru.newAtomicLRU csize else return Nothing
    return Store
        { storeDirectory  = directory
        , storeWriteAhead = writeAhead
        , storeMap        = ref
        }
  where
    csize :: Maybe Integer
csize = Integer -> Maybe Integer
forall a. a -> Maybe a
Just Integer
500

--------------------------------------------------------------------------------
withStore :: Store -> String -> (String -> FilePath -> IO a) -> [String] -> IO a
withStore :: forall a.
Store -> [Char] -> ([Char] -> [Char] -> IO a) -> [[Char]] -> IO a
withStore Store
store [Char]
loc [Char] -> [Char] -> IO a
run [[Char]]
identifier = (IOError -> IOError) -> IO a -> IO a
forall a. (IOError -> IOError) -> IO a -> IO a
modifyIOError IOError -> IOError
handle (IO a -> IO a) -> IO a -> IO a
forall a b. (a -> b) -> a -> b
$ [Char] -> [Char] -> IO a
run [Char]
key [Char]
path
  where
    key :: [Char]
key = [[Char]] -> [Char]
hash [[Char]]
identifier
    path :: [Char]
path = Store -> [Char]
storeDirectory Store
store [Char] -> ShowS
</> [Char]
key
    handle :: IOError -> IOError
handle IOError
e = IOError
e IOError -> [Char] -> IOError
`ioeSetFileName` ([Char]
path [Char] -> ShowS
forall a. [a] -> [a] -> [a]
++ [Char]
" for " [Char] -> ShowS
forall a. [a] -> [a] -> [a]
++ [Char] -> [[Char]] -> [Char]
forall a. [a] -> [[a]] -> [a]
intercalate [Char]
"/" [[Char]]
identifier)
                 IOError -> [Char] -> IOError
`ioeSetLocation` ([Char]
"Store." [Char] -> ShowS
forall a. [a] -> [a] -> [a]
++ [Char]
loc)

--------------------------------------------------------------------------------
-- | Auxiliary: add an item to the in-memory cache
cacheInsert :: Typeable a => Store -> String -> a -> IO ()
cacheInsert :: forall a. Typeable a => Store -> [Char] -> a -> IO ()
cacheInsert (Store [Char]
_ IORef (Map [Char] Box)
_ Maybe (AtomicLRU [Char] Box)
Nothing)    [Char]
_   a
_     = () -> IO ()
forall a. a -> IO a
forall (m :: * -> *) a. Monad m => a -> m a
return ()
cacheInsert (Store [Char]
_ IORef (Map [Char] Box)
_ (Just AtomicLRU [Char] Box
lru)) [Char]
key a
x =
    [Char] -> Box -> AtomicLRU [Char] Box -> IO ()
forall key val. Ord key => key -> val -> AtomicLRU key val -> IO ()
Lru.insert [Char]
key (a -> Box
forall a. Typeable a => a -> Box
Box a
x) AtomicLRU [Char] Box
lru


--------------------------------------------------------------------------------
-- | Auxiliary: get an item from the in-memory cache
cacheLookup :: forall a. Typeable a => Store -> String -> IO (Result a)
cacheLookup :: forall a. Typeable a => Store -> [Char] -> IO (Result a)
cacheLookup (Store [Char]
_ IORef (Map [Char] Box)
_ Maybe (AtomicLRU [Char] Box)
Nothing)    [Char]
_   = Result a -> IO (Result a)
forall a. a -> IO a
forall (m :: * -> *) a. Monad m => a -> m a
return Result a
forall a. Result a
NotFound
cacheLookup (Store [Char]
_ IORef (Map [Char] Box)
_ (Just AtomicLRU [Char] Box
lru)) [Char]
key = do
    res <- [Char] -> AtomicLRU [Char] Box -> IO (Maybe Box)
forall key val.
Ord key =>
key -> AtomicLRU key val -> IO (Maybe val)
Lru.lookup [Char]
key AtomicLRU [Char] Box
lru
    return $ case res of
        Maybe Box
Nothing      -> Result a
forall a. Result a
NotFound
        Just (Box a
x) -> case a -> Maybe a
forall a b. (Typeable a, Typeable b) => a -> Maybe b
cast a
x of
            Just a
x' -> a -> Result a
forall a. a -> Result a
Found a
x'
            Maybe a
Nothing -> TypeRep -> TypeRep -> Result a
forall a. TypeRep -> TypeRep -> Result a
WrongType (a -> TypeRep
forall a. Typeable a => a -> TypeRep
typeOf (a
forall a. HasCallStack => a
undefined :: a)) (a -> TypeRep
forall a. Typeable a => a -> TypeRep
typeOf a
x)


--------------------------------------------------------------------------------
cacheIsMember :: Store -> String -> IO Bool
cacheIsMember :: Store -> [Char] -> IO Bool
cacheIsMember (Store [Char]
_ IORef (Map [Char] Box)
_ Maybe (AtomicLRU [Char] Box)
Nothing)    [Char]
_   = Bool -> IO Bool
forall a. a -> IO a
forall (m :: * -> *) a. Monad m => a -> m a
return Bool
False
cacheIsMember (Store [Char]
_ IORef (Map [Char] Box)
_ (Just AtomicLRU [Char] Box
lru)) [Char]
key = Maybe Box -> Bool
forall a. Maybe a -> Bool
isJust (Maybe Box -> Bool) -> IO (Maybe Box) -> IO Bool
forall (f :: * -> *) a b. Functor f => (a -> b) -> f a -> f b
<$> [Char] -> AtomicLRU [Char] Box -> IO (Maybe Box)
forall key val.
Ord key =>
key -> AtomicLRU key val -> IO (Maybe val)
Lru.lookup [Char]
key AtomicLRU [Char] Box
lru


--------------------------------------------------------------------------------
-- | Auxiliary: delete an item from the in-memory cache
cacheDelete :: Store -> String -> IO ()
cacheDelete :: Store -> [Char] -> IO ()
cacheDelete (Store [Char]
_ IORef (Map [Char] Box)
_ Maybe (AtomicLRU [Char] Box)
Nothing)    [Char]
_   = () -> IO ()
forall a. a -> IO a
forall (m :: * -> *) a. Monad m => a -> m a
return ()
cacheDelete (Store [Char]
_ IORef (Map [Char] Box)
_ (Just AtomicLRU [Char] Box
lru)) [Char]
key = do
    _ <- [Char] -> AtomicLRU [Char] Box -> IO (Maybe Box)
forall key val.
Ord key =>
key -> AtomicLRU key val -> IO (Maybe val)
Lru.delete [Char]
key AtomicLRU [Char] Box
lru
    return ()


--------------------------------------------------------------------------------
-- | Store an item
set :: (Binary a, Typeable a) => Store -> [String] -> a -> IO ()
set :: forall a. (Binary a, Typeable a) => Store -> [[Char]] -> a -> IO ()
set Store
store [[Char]]
identifier a
value = Store -> [Char] -> ([Char] -> [Char] -> IO ()) -> [[Char]] -> IO ()
forall a.
Store -> [Char] -> ([Char] -> [Char] -> IO a) -> [[Char]] -> IO a
withStore Store
store [Char]
"set" (\[Char]
key [Char]
path -> do
    -- We need to avoid concurrent writes to the filesystem.  Imagine the
    -- follow scenario:
    --
    --  *  We compile multiple posts
    --  *  All of these fetch some common metadata
    --  *  This metadata is missing; we fetch it and then store it.
    --
    -- To solve this, we skip duplicate writes by tracking their status
    -- in 'storeWriteAhead'.  Since this set will usually be small, the
    -- required locking should be fast.  Additionally the actual IO operation
    -- still happens outside of the locking.
    first <- IORef (Map [Char] Box)
-> (Map [Char] Box -> (Map [Char] Box, Bool)) -> IO Bool
forall a b. IORef a -> (a -> (a, b)) -> IO b
IORef.atomicModifyIORef' (Store -> IORef (Map [Char] Box)
storeWriteAhead Store
store) ((Map [Char] Box -> (Map [Char] Box, Bool)) -> IO Bool)
-> (Map [Char] Box -> (Map [Char] Box, Bool)) -> IO Bool
forall a b. (a -> b) -> a -> b
$
        \Map [Char] Box
wa -> case [Char] -> Map [Char] Box -> Maybe Box
forall k a. Ord k => k -> Map k a -> Maybe a
Map.lookup [Char]
key Map [Char] Box
wa of
            Maybe Box
Nothing -> ([Char] -> Box -> Map [Char] Box -> Map [Char] Box
forall k a. Ord k => k -> a -> Map k a -> Map k a
Map.insert [Char]
key (a -> Box
forall a. Typeable a => a -> Box
Box a
value) Map [Char] Box
wa, Bool
True)
            Just Box
_  -> (Map [Char] Box
wa, Bool
False)

    cacheInsert store key value

    -- Only the thread that stored the writeAhead should actually write this
    -- file.  That way, only one thread at a time will try to write this.
    -- Release the writeAhead value once we're done.
    when first $ do
        encodeFile path value
        IORef.atomicModifyIORef' (storeWriteAhead store) $
            \Map [Char] Box
wa -> ([Char] -> Map [Char] Box -> Map [Char] Box
forall k a. Ord k => k -> Map k a -> Map k a
Map.delete [Char]
key Map [Char] Box
wa, ())
  ) [[Char]]
identifier


--------------------------------------------------------------------------------
-- | Load an item
get :: forall a. (Binary a, Typeable a) => Store -> [String] -> IO (Result a)
get :: forall a.
(Binary a, Typeable a) =>
Store -> [[Char]] -> IO (Result a)
get Store
store = Store
-> [Char]
-> ([Char] -> [Char] -> IO (Result a))
-> [[Char]]
-> IO (Result a)
forall a.
Store -> [Char] -> ([Char] -> [Char] -> IO a) -> [[Char]] -> IO a
withStore Store
store [Char]
"get" (([Char] -> [Char] -> IO (Result a)) -> [[Char]] -> IO (Result a))
-> ([Char] -> [Char] -> IO (Result a)) -> [[Char]] -> IO (Result a)
forall a b. (a -> b) -> a -> b
$ \[Char]
key [Char]
path -> do
    -- Check the writeAhead value
    writeAhead <- IORef (Map [Char] Box) -> IO (Map [Char] Box)
forall a. IORef a -> IO a
IORef.readIORef (IORef (Map [Char] Box) -> IO (Map [Char] Box))
-> IORef (Map [Char] Box) -> IO (Map [Char] Box)
forall a b. (a -> b) -> a -> b
$ Store -> IORef (Map [Char] Box)
storeWriteAhead Store
store
    case Map.lookup key writeAhead of
        Just (Box a
x) -> case a -> Maybe a
forall a b. (Typeable a, Typeable b) => a -> Maybe b
cast a
x of
            Just a
x' -> Result a -> IO (Result a)
forall a. a -> IO a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (Result a -> IO (Result a)) -> Result a -> IO (Result a)
forall a b. (a -> b) -> a -> b
$ a -> Result a
forall a. a -> Result a
Found a
x'
            Maybe a
Nothing -> Result a -> IO (Result a)
forall a. a -> IO a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (Result a -> IO (Result a)) -> Result a -> IO (Result a)
forall a b. (a -> b) -> a -> b
$ TypeRep -> TypeRep -> Result a
forall a. TypeRep -> TypeRep -> Result a
WrongType (a -> TypeRep
forall a. Typeable a => a -> TypeRep
typeOf (a
forall a. HasCallStack => a
undefined :: a)) (a -> TypeRep
forall a. Typeable a => a -> TypeRep
typeOf a
x)
        Maybe Box
Nothing -> do
            -- Check the in-memory map
            ref <- Store -> [Char] -> IO (Result a)
forall a. Typeable a => Store -> [Char] -> IO (Result a)
cacheLookup Store
store [Char]
key
            case ref of
                -- Not found in the map, try the filesystem
                Result a
NotFound -> do
                    exists <- [Char] -> IO Bool
doesFileExist [Char]
path
                    if not exists
                        -- Not found in the filesystem either
                        then return NotFound
                        -- Found in the filesystem
                        else do
                            v <- decodeClose path
                            cacheInsert store key v
                            return $ Found v
                -- Found in the in-memory map (or wrong type), just return
                Result a
s -> Result a -> IO (Result a)
forall a. a -> IO a
forall (m :: * -> *) a. Monad m => a -> m a
return Result a
s
  where
    -- 'decodeFile' from Data.Binary which closes the file ASAP
    decodeClose :: [Char] -> IO b
decodeClose [Char]
path = do
        h   <- [Char] -> IOMode -> IO Handle
openFile [Char]
path IOMode
ReadMode
        lbs <- BL.hGetContents h
        BL.length lbs `seq` hClose h
        return $ decode lbs


--------------------------------------------------------------------------------
-- | Strict function
isMember :: Store -> [String] -> IO Bool
isMember :: Store -> [[Char]] -> IO Bool
isMember Store
store = Store
-> [Char] -> ([Char] -> [Char] -> IO Bool) -> [[Char]] -> IO Bool
forall a.
Store -> [Char] -> ([Char] -> [Char] -> IO a) -> [[Char]] -> IO a
withStore Store
store [Char]
"isMember" (([Char] -> [Char] -> IO Bool) -> [[Char]] -> IO Bool)
-> ([Char] -> [Char] -> IO Bool) -> [[Char]] -> IO Bool
forall a b. (a -> b) -> a -> b
$ \[Char]
key [Char]
path -> do
    writeAhead <- IORef (Map [Char] Box) -> IO (Map [Char] Box)
forall a. IORef a -> IO a
IORef.readIORef (IORef (Map [Char] Box) -> IO (Map [Char] Box))
-> IORef (Map [Char] Box) -> IO (Map [Char] Box)
forall a b. (a -> b) -> a -> b
$ Store -> IORef (Map [Char] Box)
storeWriteAhead Store
store
    if Map.member key writeAhead
        then pure True
        else do
            inCache <- cacheIsMember store key
            if inCache then return True else doesFileExist path


--------------------------------------------------------------------------------
-- | Delete an item
delete :: Store -> [String] -> IO ()
delete :: Store -> [[Char]] -> IO ()
delete Store
store = Store -> [Char] -> ([Char] -> [Char] -> IO ()) -> [[Char]] -> IO ()
forall a.
Store -> [Char] -> ([Char] -> [Char] -> IO a) -> [[Char]] -> IO a
withStore Store
store [Char]
"delete" (([Char] -> [Char] -> IO ()) -> [[Char]] -> IO ())
-> ([Char] -> [Char] -> IO ()) -> [[Char]] -> IO ()
forall a b. (a -> b) -> a -> b
$ \[Char]
key [Char]
path -> do
    Store -> [Char] -> IO ()
cacheDelete Store
store [Char]
key
    [Char] -> IO ()
deleteFile [Char]
path


--------------------------------------------------------------------------------
-- | Delete a file unless it doesn't exist...
deleteFile :: FilePath -> IO ()
deleteFile :: [Char] -> IO ()
deleteFile = (IO () -> (IOError -> IO ()) -> IO ()
forall a. IO a -> (IOError -> IO a) -> IO a
`catchIOError` \IOError
_ -> () -> IO ()
forall a. a -> IO a
forall (m :: * -> *) a. Monad m => a -> m a
return ()) (IO () -> IO ()) -> ([Char] -> IO ()) -> [Char] -> IO ()
forall b c a. (b -> c) -> (a -> b) -> a -> c
. [Char] -> IO ()
removeFile


--------------------------------------------------------------------------------
-- | Mostly meant for internal usage
hash :: [String] -> String
hash :: [[Char]] -> [Char]
hash = Int -> [Char]
forall a. Show a => a -> [Char]
show (Int -> [Char]) -> ([[Char]] -> Int) -> [[Char]] -> [Char]
forall b c a. (b -> c) -> (a -> b) -> a -> c
. [Char] -> Int
forall a. Hashable a => a -> Int
DH.hash ([Char] -> Int) -> ([[Char]] -> [Char]) -> [[Char]] -> Int
forall b c a. (b -> c) -> (a -> b) -> a -> c
. [Char] -> [[Char]] -> [Char]
forall a. [a] -> [[a]] -> [a]
intercalate [Char]
"/"