View source for HXT/Practical/Google1

<hask>
{-# LANGUAGE Arrows, NoMonomorphismRestriction #-}
import Text.XML.HXT.Arrow
import Text.Printf
import Network.URI
import Network.HTTP
import Data.List
import Data.Char
import System.Environment

-- Search Google and retrieve the results into some
-- nicer data structure.  This is an exercise in
-- munging the HTML output of Google.

-- A number of possible functions are demonstrated,
-- to show some different ways of going about the task,
-- only one is really needed.

-- Personally, I favor the 2nd version, selectGoogleResultsToXML.  
-- I think this is a good demonstration of a case where the 
-- ordinary Haskell combinators are clearer than the special 
-- Arrow syntax.

--------------------------------------------------
-- First example: Select pairs of (url, title)

-- There isn't any help, from Google, in picking out the results,
-- so I've poked around and came up with this:
-- look for 
--   <div id="res"> ... <h2> ... <a onmousedown="..." ...>
-- and the results will be contained within the anchor.

selectGoogleResults = 
  atTagCase "div" >>> hasAttrValue "id" (=="res") 
  >>> atTagCase "h2"
  >>> atTagCase "a" >>> hasAttrValue "onmousedown" (not . null) 
  >>> (getAttrValue "href" &&& 
       -- The title may be broken up into multiple text nodes.
       -- So, we collect it as a list and then lift 'concat' 
       -- to combine it.
       (listA (deep isText >>> getText) >>> arr concat))

-- Or, select the data into a simpler XML document

selectGoogleResultsToXML = 
  selem "results" 
    [ atTagCase "div" >>> hasAttrValue "id" (=="res") 
      >>> atTagCase "h2"
      >>> atTagCase "a" >>> hasAttrValue "onmousedown" (not . null) 
      >>> selem "result" 
            [ selem "url" [getAttrValue "href" >>> mkText]
            , selem "title" [deep isText] ] ] 

-- The above function shows construction of XML,
-- 
-- selem tag body = mkelem tag [] body
--   constructs an element without attributes.
--
-- Note that 'body' is a list containing XmlTree arrows,
-- that is why you can mix selection and construction.

--------------------------------------------------
-- Alternative versions, using the special Arrow 
-- syntax.

selectGoogleResults' = 
  atTagCase "div" >>> hasAttrValue "id" (=="res") 
  >>> atTagCase "h2"
  >>> atTagCase "a" >>> hasAttrValue "onmousedown" (not . null) 
  >>> proc r -> do
        url   <- getAttrValue "href"             -< r
        title <- listA (getText <<< deep isText) -< r
        returnA -< (url, concat title)

selectGoogleResultsToXML' = proc x -> do
  res <- listA (atTagCase "div" >>> hasAttrValue "id" (=="res")
                >>> atTagCase "h2"
                >>> atTagCase "a" >>> hasAttrValue "onmousedown" (not . null)
                >>> selectResult) -< x
  selem "results" (map constA res) -<< ()
  where
    selectResult = proc r -> do
      url   <- getAttrValue "href"             -< r
      title <- listA (getText <<< deep isText) -< r
      selem "result" 
        [ selem "url" [txt url]
        , selem "title" [txt (concat title)] ]  -<< ()

-- This last example introduces some new syntax,
-- namely -<<.  According to the GHC Arrow docs,
-- you cannot use locally bound variables on the
-- left-hand side of -<.  In brief, the -<< syntax is
-- a variation which permits this, while introducing
-- a dependency on the ArrowApply class.

-- The reason why 'selem' is on the LHS in the
-- first place is because, if you check the type
-- of 'selem', you will see that it is already
-- in the Arrow (so to speak).  Injecting it
-- with 'returnA' would simply create an extra
-- layer of Arrow that is not wanted.

-- Since 'selem' is not processing anything, the RHS 
-- of -<< in this case is simply ().

googleURLFormat = "http://www.google.com/search?%s"
constructGoogleURL q = printf googleURLFormat $ urlEncodeVars [("q",q)]

-- case-insensitive tag matching
atTagCase tag = deep (isElem >>> hasNameWith ((== tag') . upper . localPart))
  where tag' = upper tag
        upper = map toUpper

parseHTML = readString [(a_validate,v_0),(a_parse_html,v_1),(a_issue_warnings,v_0)]

-- Pretend to be a user of Mozilla Firefox, because Google
-- will not display results for unknown user agents.

userAgent = "Mozilla/5.0 (en-US) Firefox/2.0.0.6667" 

get :: URI -> IO String
get uri = do
  let req = Request uri GET [] ""
  eresp <- simpleHTTP $ insertHeader HdrUserAgent userAgent req
  case eresp of
    Left er -> error $ show er
    Right res -> return $ rspBody res

main = do
  args <- getArgs
  case parseURI (constructGoogleURL (unwords args)) of
    Nothing -> putStrLn "Invalid search"
    Just uri -> do
      body  <- get uri

      -- tuple version
      links <- runX (parseHTML body >>> selectGoogleResults)
      flip mapM_ links $ \ (url,title) -> printf "%s <url:%s>\n" title url

      -- XML version
      [xml] <- runX (parseHTML body >>> 
                     -- At the top of a document is a hidden "root" node
                     -- which encompasses all the top-level siblings.
                     root [] [selectGoogleResultsToXML] >>> 
                     writeDocumentToString [(a_indent,v_1)])
      putStrLn xml
</hask>