Tests comparing against cardano-cli, on top of cardano-testnet and tx-generator

flake.nix

@@ -14,6 +14,7 @@
       inputs.nixpkgs.follows = "nixpkgs";
       inputs.hackage.follows = "hackageNix";
     };
+    cardano-node.url = "github:IntersectMBO/cardano-node/11.0.1";
     iohkNix.url = "github:input-output-hk/iohk-nix";
     nixpkgs.follows = "haskellNix/nixpkgs-unstable";
     self.submodules = true;
@@ -27,7 +28,7 @@
       systems = [
         "x86_64-linux"
         # "aarch64-linux"
-        # "aarch64-darwin"
+        "aarch64-darwin"
       ];
       perSystem = {system, ...}: {
         _module.args.pkgs = import inputs.nixpkgs {

perSystem/devShells.nix

@@ -1,5 +1,5 @@
 { inputs, ... }: {
-  perSystem = { shellFor, pkgs, ... }: {
+  perSystem = { shellFor, hsPkgs, pkgs, system, ... }: {
     devShells.default = shellFor {
       packages = p: [ p.ogmios ];
 
@@ -29,5 +29,28 @@
 
       withHoogle = true;
     };
+
+    devShells.integration = let
+      cn = inputs.cardano-node.packages.${system};
+    in shellFor {
+      packages = p: [ p.ogmios p.ogmios-integration-tests ];
+
+      nativeBuildInputs = [
+        pkgs.jq
+        hsPkgs.ogmios.components.exes.ogmios
+        cn.cardano-node
+        cn.cardano-cli
+        cn.cardano-testnet
+        cn.tx-generator
+      ];
+
+      tools = {
+        cabal = "latest";
+      };
+
+      shellHook = ''
+        export LANG="en_US.UTF-8"
+      '';
+    };
   };
 }

perSystem/packages.nix

@@ -8,6 +8,7 @@
         default = ogmios.components.exes.ogmios;
         ogmios = ogmios.components.exes.ogmios;
         ogmios-lib = ogmios.components.library;
+        ogmios-integration-tests = hsPkgs.ogmios-integration-tests.components.exes.ogmios-integration-tests;
       };
 
       checks.ogmios-unit = ogmios.checks.unit;

server/cabal.project

@@ -46,6 +46,7 @@ packages:
   modules/hjsonschema
   modules/hspec-json-schema
   modules/json-rpc
+  test/integration
 
 tests: False
 

server/test/integration/src/Main.hs

@@ -0,0 +1,30 @@
+module Main (main) where
+
+import Test.Tasty (defaultMain, testGroup)
+import Test.Integration.Constitution (constitutionTests)
+import Test.Integration.DelegateRepresentatives (delegateRepresentativesTests)
+import Test.Integration.Env (withTestEnv)
+import Test.Integration.Epoch (epochTests)
+import Test.Integration.LedgerTip (ledgerTipTests)
+import Test.Integration.LiveStakeDistribution (liveStakeDistributionTests)
+import Test.Integration.NetworkTip (networkTipTests)
+import Test.Integration.ProtocolParameters (protocolParametersTests)
+import Test.Integration.RewardAccountSummaries (rewardAccountSummariesTests)
+import Test.Integration.StakePools (stakePoolsTests)
+import Test.Integration.Utxo (utxoTests)
+
+main :: IO ()
+main = defaultMain $
+  withTestEnv $ \getEnv ->
+    testGroup "Ogmios Integration Tests"
+      [ utxoTests getEnv
+      , protocolParametersTests getEnv
+      , ledgerTipTests getEnv
+      , networkTipTests getEnv
+      , epochTests getEnv
+      , stakePoolsTests getEnv
+      , rewardAccountSummariesTests getEnv
+      , liveStakeDistributionTests getEnv
+      , constitutionTests getEnv
+      , delegateRepresentativesTests getEnv
+      ]

server/test/integration/src/Test/Integration/Constitution.hs

@@ -0,0 +1,100 @@
+{-# LANGUAGE OverloadedStrings #-}
+{-# LANGUAGE ScopedTypeVariables #-}
+
+module Test.Integration.Constitution
+  ( constitutionTests
+  ) where
+
+import Data.Aeson
+    ( Value(..)
+    , (.:)
+    , eitherDecode
+    , encode
+    , object
+    , (.=)
+    )
+import Data.Aeson.Types (parseEither, withObject)
+import Data.Text (Text)
+import System.FilePath ((</>))
+import System.Process (callProcess)
+import Test.Tasty (TestTree, testGroup)
+import Test.Tasty.HUnit (assertEqual, assertFailure, testCase)
+
+import qualified Data.Aeson.KeyMap as KM
+import qualified Data.ByteString.Lazy as LBS
+import qualified Network.WebSockets as WS
+
+import Test.Integration.Env (TestEnv(..), queryOgmiosRetry)
+
+constitutionTests :: IO TestEnv -> TestTree
+constitutionTests getEnv = testGroup "Constitution"
+  [ testCase "constitution matches cardano-cli" $ do
+      env <- getEnv
+
+      ogmiosResp <- queryOgmiosRetry (envOgmiosPort env) queryOgmios
+      ogmiosResult <- case ogmiosResp of
+        Object o
+          | Just result <- KM.lookup "result" o -> pure result
+          | Just err    <- KM.lookup "error"  o ->
+              assertFailure $ "Ogmios returned error: " <> show err
+        _ -> assertFailure $ "Unexpected ogmios response: " <> show ogmiosResp
+
+      cliResult <- queryCli (envWorkDir env) (envNodeSocket env) (envTestnetMagic env)
+
+      oHash <- parseOgmiosHash ogmiosResult
+      cHash <- parseCliHash cliResult
+      assertEqual "constitution anchor hash" oHash cHash
+  ]
+
+-- ---------------------------------------------------------------------------
+-- Queries
+-- ---------------------------------------------------------------------------
+
+queryOgmios :: Int -> IO Value
+queryOgmios port =
+  WS.runClient "127.0.0.1" port "/" $ \conn -> do
+    WS.sendTextData conn $ encode $ object
+      [ "jsonrpc" .= ("2.0" :: Text)
+      , "method"  .= ("queryLedgerState/constitution" :: Text)
+      , "id"      .= Null
+      ]
+    resp <- WS.receiveData conn
+    case eitherDecode resp of
+      Left err  -> fail $ "Failed to decode ogmios response: " <> err
+      Right val -> pure val
+
+queryCli :: FilePath -> FilePath -> Int -> IO Value
+queryCli workDir socketPath magic = do
+  let outFile = workDir </> "cli-constitution.json"
+  callProcess "cardano-cli"
+    [ "conway", "query", "constitution"
+    , "--testnet-magic", show magic
+    , "--socket-path", socketPath
+    , "--out-file", outFile
+    ]
+  contents <- LBS.readFile outFile
+  case eitherDecode contents of
+    Left err  -> fail $ "Failed to decode cardano-cli output: " <> err
+    Right val -> pure val
+
+-- ---------------------------------------------------------------------------
+-- Parsers
+-- ---------------------------------------------------------------------------
+
+parseOgmiosHash :: Value -> IO Text
+parseOgmiosHash val = case parseEither parser val of
+    Left err -> assertFailure $ "Failed to parse ogmios constitution: " <> err
+    Right v  -> pure v
+  where
+    parser = withObject "constitution" $ \o -> do
+      metadata <- o .: "metadata"
+      metadata .: "hash"
+
+parseCliHash :: Value -> IO Text
+parseCliHash val = case parseEither parser val of
+    Left err -> assertFailure $ "Failed to parse cli constitution: " <> err
+    Right v  -> pure v
+  where
+    parser = withObject "constitution" $ \o -> do
+      anchor <- o .: "anchor"
+      anchor .: "dataHash"

server/test/integration/src/Test/Integration/DelegateRepresentatives.hs

@@ -0,0 +1,126 @@
+{-# LANGUAGE OverloadedStrings #-}
+{-# LANGUAGE ScopedTypeVariables #-}
+
+module Test.Integration.DelegateRepresentatives
+  ( delegateRepresentativesTests
+  ) where
+
+import Control.Monad (when)
+import Data.Aeson
+    ( Value(..)
+    , eitherDecode
+    , encode
+    , object
+    , (.=)
+    )
+import Data.Aeson.Types (parseEither, withArray, withObject)
+import Data.Foldable (toList)
+import Data.Set (Set)
+import Data.Text (Text)
+import System.FilePath ((</>))
+import System.Process (callProcess)
+import Test.Tasty (TestTree, testGroup)
+import Test.Tasty.HUnit (assertFailure, testCase)
+
+import qualified Data.Aeson.KeyMap as KM
+import qualified Data.ByteString.Lazy as LBS
+import qualified Data.Set as Set
+import qualified Network.WebSockets as WS
+
+import Test.Integration.Env (TestEnv(..), queryOgmiosRetry)
+
+delegateRepresentativesTests :: IO TestEnv -> TestTree
+delegateRepresentativesTests getEnv = testGroup "DelegateRepresentatives"
+  [ testCase "DRep IDs match cardano-cli" $ do
+      env <- getEnv
+
+      ogmiosResp <- queryOgmiosRetry (envOgmiosPort env) queryOgmios
+      ogmiosResult <- case ogmiosResp of
+        Object o
+          | Just result <- KM.lookup "result" o -> pure result
+          | Just err    <- KM.lookup "error"  o ->
+              assertFailure $ "Ogmios returned error: " <> show err
+        _ -> assertFailure $ "Unexpected ogmios response: " <> show ogmiosResp
+
+      cliResult <- queryCli (envWorkDir env) (envNodeSocket env) (envTestnetMagic env)
+
+      ogmiosDreps <- case parseOgmiosDrepIds ogmiosResult of
+        Left err -> assertFailure $ "Failed to parse ogmios DRep IDs: " <> err
+        Right s  -> pure s
+      cliDreps <- case parseCliDrepIds cliResult of
+        Left err -> assertFailure $ "Failed to parse cli DRep IDs: " <> err
+        Right s  -> pure s
+
+      let ogmiosOnly = Set.difference ogmiosDreps cliDreps
+          cliOnly    = Set.difference cliDreps ogmiosDreps
+      when (not (Set.null ogmiosOnly) || not (Set.null cliOnly)) $
+        assertFailure $ unlines
+          [ "DRep ID sets differ:"
+          , "  In Ogmios only: " <> show (Set.toList ogmiosOnly)
+          , "  In cardano-cli only: " <> show (Set.toList cliOnly)
+          ]
+  ]
+
+-- ---------------------------------------------------------------------------
+-- Queries
+-- ---------------------------------------------------------------------------
+
+queryOgmios :: Int -> IO Value
+queryOgmios port =
+  WS.runClient "127.0.0.1" port "/" $ \conn -> do
+    WS.sendTextData conn $ encode $ object
+      [ "jsonrpc" .= ("2.0" :: Text)
+      , "method"  .= ("queryLedgerState/delegateRepresentatives" :: Text)
+      , "id"      .= Null
+      ]
+    resp <- WS.receiveData conn
+    case eitherDecode resp of
+      Left err  -> fail $ "Failed to decode ogmios response: " <> err
+      Right val -> pure val
+
+queryCli :: FilePath -> FilePath -> Int -> IO Value
+queryCli workDir socketPath magic = do
+  let outFile = workDir </> "cli-drep-state.json"
+  callProcess "cardano-cli"
+    [ "conway", "query", "drep-state"
+    , "--all-dreps"
+    , "--testnet-magic", show magic
+    , "--socket-path", socketPath
+    , "--out-file", outFile
+    ]
+  contents <- LBS.readFile outFile
+  case eitherDecode contents of
+    Left err  -> fail $ "Failed to decode cardano-cli output: " <> err
+    Right val -> pure val
+
+-- ---------------------------------------------------------------------------
+-- Parsers
+-- ---------------------------------------------------------------------------
+
+-- Ogmios returns an array of objects with "id" and "type" fields.
+-- Special entries "abstain" and "noConfidence" have no "id".
+parseOgmiosDrepIds :: Value -> Either String (Set Text)
+parseOgmiosDrepIds = parseEither $ withArray "dreps" $ \arr -> do
+  ids <- mapM extractId (toList arr)
+  pure $ Set.fromList [i | Just i <- ids]
+  where
+    extractId = withObject "drep" $ \o ->
+      case KM.lookup "id" o of
+        Just (String drepId) -> pure (Just drepId)
+        _ -> pure Nothing
+
+-- cardano-cli drep-state --all-dreps returns an array of [drepId, drepState] pairs
+parseCliDrepIds :: Value -> Either String (Set Text)
+parseCliDrepIds = parseEither $ withArray "dreps" $ \arr -> do
+  ids <- mapM extractId (toList arr)
+  pure (Set.fromList ids)
+  where
+    extractId = withArray "pair" $ \pair -> case toList pair of
+      (drepId:_) -> case drepId of
+        Object o -> case KM.lookup "keyHash" o of
+          Just (String h) -> pure h
+          _ -> case KM.lookup "scriptHash" o of
+            Just (String h) -> pure h
+            _ -> fail $ "No keyHash or scriptHash in DRep ID: " <> show o
+        _ -> fail $ "Expected object for DRep ID, got: " <> show drepId
+      _ -> fail "Empty DRep pair"