Add --convert option from Hoogle4

docs/Install.md

@@ -29,6 +29,9 @@ Run `hoogle generate --local` to query `ghc-pkg` and generate links for all pack
 
 Run `hoogle generate --local=mydir` to generate an index for the packages in `mydir`, which must contain `foo.txt` Hoogle input files. Links to the results will default to Hackage, but if `@url` directives are in the `.txt` files they can override the link destination.
 
+### Index a text database
+Run `hoogle generate --convert=MyPackage.txt` to generate an index for one Hoogle-formated textual database. Do not expect links in the results to work, this mode is meant for CLI search. Such a database can be generated with `haddock --hoogle --package-name=MyPackage --odir=<directory of your choosing> <haskell files to be hooglified>`
+
 ## Searching a Hoogle database
 
 You can run searches on the command line or by spawning a web server.
@@ -45,7 +48,7 @@ If your database points at the local file system pass `--local` to reserve `file
 
 ### GHCi Integration
 
-Ever feel like having access to hoogle whilst messing around in GHCi? It's relatively easy to integrate the two. 
+Ever feel like having access to hoogle whilst messing around in GHCi? It's relatively easy to integrate the two.
 
 The following will install hoogle as a shell command, and configure GHCi to have a command ":hoogle":
 
@@ -62,9 +65,9 @@ Done!
 On Windows you should add the same line
  :def hoogle \x -> return $ ":!hoogle \"" ++ x ++ "\""
 to file (XP/2003):
- C:\Documents and Settings\[your windows account]\Application Data\ghc\ghci.conf 
+ C:\Documents and Settings\[your windows account]\Application Data\ghc\ghci.conf
 or(Windows Vista/7):
- C:\users\[your windows account]\Application Data\ghc\ghci.conf 
+ C:\users\[your windows account]\Application Data\ghc\ghci.conf
 
 #### How it works
 

src/Action/CmdLine.hs

@@ -37,6 +37,7 @@ data CmdLine
     | Generate
         {download :: Maybe Bool
         ,database :: FilePath
+        ,convert :: [FilePath]
         ,insecure :: Bool
         ,include :: [String]
         ,count :: Maybe Int
@@ -127,6 +128,7 @@ search_ = Search
 generate = Generate
     {download = def &= help "Download all files from the web"
     ,insecure = def &= help "Allow insecure HTTPS connections"
+    ,convert = def &=help "Convert a set of textbases to the hoo format."
     ,include = def &= args &= typ "PACKAGE"
     ,local_ = def &= opt "" &= help "Index local packages and link to local haddock docs"
     ,count = Nothing &= name "n" &= help "Maximum number of packages to index (defaults to all)"

src/Action/Generate.hs

@@ -121,6 +121,10 @@ readHaskellOnline timing settings download = do
     pure (cbl, want, source)
 
 
+-- | readHaskellDirs will look for .txt files anywhere under @dirs@.
+-- It uses these local files as sources for indexing.
+-- Links to the results will default to Hackage, but if @url directives
+-- are in the .txt files they can override the link destination.
 readHaskellDirs :: Timing -> Settings -> [FilePath] -> IO (Map.Map PkgName Package, Set.Set PkgName, ConduitT () (PkgName, URL, LBStr) IO ())
 readHaskellDirs timing settings dirs = do
     files <- concatMapM listFilesRecursive dirs
@@ -161,6 +165,8 @@ readFregeOnline timing download = do
     pure (Map.empty, Set.singleton $ strPack "frege", source)
 
 
+-- | readHaskellGhcpkg will use every installed package (info via ghc-pkg)
+-- as a source. It uses each package's documentation directory as source for indexing.
 readHaskellGhcpkg :: Timing -> Settings -> IO (Map.Map PkgName Package, Set.Set PkgName, ConduitT () (PkgName, URL, LBStr) IO ())
 readHaskellGhcpkg timing settings = do
     cbl <- timed timing "Reading ghc-pkg" $ readGhcPkg settings
@@ -177,6 +183,26 @@ readHaskellGhcpkg timing settings = do
                     in Map.map (\p -> p{packageTags = ts ++ packageTags p}) cbl
     pure (cbl, Map.keysSet cbl, source)
 
+-- | readHaskellHaddockFile will read a list of text databases, foo.txt, and use
+-- them as sources for indexing.
+readHaskellHaddockFile :: Timing -> Settings -> [FilePath] -> IO (Map.Map PkgName Package, Set.Set PkgName, ConduitT () (PkgName, URL, LBStr) IO ())
+readHaskellHaddockFile timing settings txtdbs = do
+    let source =
+            forM_ txtdbs $ \txt -> do
+                let name = strPack $ take (length txt - (length ".txt")) txt
+                whenM (liftIO $ doesFileExist txt) $ do
+                    src <- liftIO $ bstrReadFile txt
+                    let url = "file://./" ++ txt
+                    yield (name, url, lbstrFromChunks [src])
+    let pkgs = Set.fromList $ map (\txt -> strPack $ take (length txt - (length ".txt")) txt) txtdbs
+    pure (Map.empty, pkgs, source)
+
+    where docDir name Package{..} = name ++ "-" ++ strUnpack packageVersion
+
+-- | readHaskellHaddock takes a @baseDocDir@, which is the filepath prefix for
+-- ghc-pkg's documentation directories. This function reads all installed
+-- packages on the system (via ghc-pkg), it tries to read all files matching
+-- @baseDocDir@/@<pkgname>-<version>@/@<pkgname>.txt@
 readHaskellHaddock :: Timing -> Settings -> FilePath -> IO (Map.Map PkgName Package, Set.Set PkgName, ConduitT () (PkgName, URL, LBStr) IO ())
 readHaskellHaddock timing settings docBaseDir = do
     cbl <- timed timing "Reading ghc-pkg" $ readGhcPkg settings
@@ -204,7 +230,8 @@ actionGenerate g@Generate{..} = withTiming (if debug then Just $ replaceExtensio
     download <- pure $ downloadInput timing insecure download (takeDirectory database)
     settings <- loadSettings
     (cbl, want, source) <- case language of
-        Haskell | Just dir <- haddock -> readHaskellHaddock timing settings dir
+        Haskell | txtdbs@(_:_) <- convert -> readHaskellHaddockFile timing settings txtdbs
+                | Just dir <- haddock -> readHaskellHaddock timing settings dir
                 | [""] <- local_ -> readHaskellGhcpkg timing settings
                 | [] <- local_ -> readHaskellOnline timing settings download
                 | otherwise -> readHaskellDirs timing settings local_