add an api to dump postgres schema (close #1939) (#1967)

This commit is contained in:
Shahidh K Muhammed
2019-04-30 14:04:08 +05:30
committed by Vamshi Surabhi
parent 8389a7e273
commit 71cf017197
31 changed files with 750 additions and 159 deletions

View File

@@ -7,6 +7,7 @@ nproc := $(shell nproc)
# TODO: needs to be replaced with something like yq
stack_resolver := $(shell awk '/^resolver:/ {print $$2;}' stack.yaml)
packager_ver := 20190326
pg_dump_ver := 11
project_dir := $(shell pwd)
build_dir := $(project_dir)/$(shell stack path --dist-dir)/build
@@ -59,6 +60,7 @@ ci-image:
docker cp $(build_dir)/$(project)/$(project) dummy:/root/
docker run --rm --volumes-from dummy $(registry)/graphql-engine-packager:$(packager_ver) /build.sh $(project) | tar -x -C packaging/build/rootfs
strip --strip-unneeded packaging/build/rootfs/bin/$(project)
cp /usr/lib/postgresql/$(pg_dump_ver)/bin/pg_dump packaging/build/rootfs/bin/pg_dump
upx packaging/build/rootfs/bin/$(project)
docker build -t $(registry)/$(project):$(VERSION) packaging/build/

View File

@@ -165,6 +165,7 @@ library
, Hasura.Server.CheckUpdates
, Hasura.Server.Telemetry
, Hasura.Server.SchemaUpdate
, Hasura.Server.PGDump
, Hasura.RQL.Types
, Hasura.RQL.Instances
, Hasura.RQL.Types.SchemaCache

View File

@@ -144,7 +144,7 @@ main = do
prepareEvents logger ci
(app, cacheRef, cacheInitTime) <-
mkWaiApp isoL loggerCtx sqlGenCtx pool httpManager am
mkWaiApp isoL loggerCtx sqlGenCtx pool ci httpManager am
corsCfg enableConsole enableTelemetry instanceId enabledAPIs lqOpts
-- log inconsistent schema objects

View File

@@ -52,6 +52,7 @@ import Hasura.Server.Cors
import Hasura.Server.Init
import Hasura.Server.Logging
import Hasura.Server.Middleware (corsMiddleware)
import qualified Hasura.Server.PGDump as PGD
import Hasura.Server.Query
import Hasura.Server.Utils
import Hasura.Server.Version
@@ -130,16 +131,17 @@ withSCUpdate scr logger action = do
data ServerCtx
= ServerCtx
{ scPGExecCtx :: PGExecCtx
, scLogger :: L.Logger
, scCacheRef :: SchemaCacheRef
, scAuthMode :: AuthMode
, scManager :: HTTP.Manager
, scSQLGenCtx :: SQLGenCtx
, scEnabledAPIs :: S.HashSet API
, scInstanceId :: InstanceId
, scPlanCache :: E.PlanCache
, scLQState :: EL.LiveQueriesState
{ scPGExecCtx :: PGExecCtx
, scConnInfo :: Q.ConnInfo
, scLogger :: L.Logger
, scCacheRef :: SchemaCacheRef
, scAuthMode :: AuthMode
, scManager :: HTTP.Manager
, scSQLGenCtx :: SQLGenCtx
, scEnabledAPIs :: S.HashSet API
, scInstanceId :: InstanceId
, scPlanCache :: E.PlanCache
, scLQState :: EL.LiveQueriesState
}
data HandlerCtx
@@ -152,12 +154,27 @@ data HandlerCtx
type Handler = ExceptT QErr (ReaderT HandlerCtx IO)
data APIResp
= JSONResp !EncJSON
| RawResp !T.Text !BL.ByteString -- content-type, body
apiRespToLBS :: APIResp -> BL.ByteString
apiRespToLBS = \case
JSONResp j -> encJToLBS j
RawResp _ b -> b
mkAPIRespHandler :: Handler EncJSON -> Handler APIResp
mkAPIRespHandler = fmap JSONResp
isMetadataEnabled :: ServerCtx -> Bool
isMetadataEnabled sc = S.member METADATA $ scEnabledAPIs sc
isGraphQLEnabled :: ServerCtx -> Bool
isGraphQLEnabled sc = S.member GRAPHQL $ scEnabledAPIs sc
isPGDumpEnabled :: ServerCtx -> Bool
isPGDumpEnabled sc = S.member PGDUMP $ scEnabledAPIs sc
isDeveloperAPIEnabled :: ServerCtx -> Bool
isDeveloperAPIEnabled sc = S.member DEVELOPER $ scEnabledAPIs sc
@@ -204,7 +221,7 @@ mkSpockAction
:: (MonadIO m)
=> (Bool -> QErr -> Value)
-> ServerCtx
-> Handler EncJSON
-> Handler APIResp
-> ActionT m ()
mkSpockAction qErrEncoder serverCtx handler = do
req <- request
@@ -219,14 +236,13 @@ mkSpockAction qErrEncoder serverCtx handler = do
let handlerState = HandlerCtx serverCtx reqBody userInfo headers
t1 <- liftIO getCurrentTime -- for measuring response time purposes
result <- liftIO $ runReaderT (runExceptT handler) handlerState
eResult <- liftIO $ runReaderT (runExceptT handler) handlerState
t2 <- liftIO getCurrentTime -- for measuring response time purposes
let resLBS = fmap encJToLBS result
-- log result
logResult (Just userInfo) req reqBody serverCtx resLBS $ Just (t1, t2)
either (qErrToResp $ userRole userInfo == adminRole) resToResp resLBS
logResult (Just userInfo) req reqBody serverCtx (apiRespToLBS <$> eResult) $ Just (t1, t2)
either (qErrToResp $ userRole userInfo == adminRole) resToResp eResult
where
logger = scLogger serverCtx
@@ -240,9 +256,14 @@ mkSpockAction qErrEncoder serverCtx handler = do
logError Nothing req reqBody serverCtx qErr
qErrToResp includeInternal qErr
resToResp resp = do
uncurry setHeader jsonHeader
lazyBytes resp
resToResp eResult = do
case eResult of
JSONResp j -> do
uncurry setHeader jsonHeader
lazyBytes $ encJToLBS j
RawResp ct b -> do
setHeader "content-type" ct
lazyBytes b
v1QueryHandler :: RQLQuery -> Handler EncJSON
v1QueryHandler query = do
@@ -293,6 +314,13 @@ gqlExplainHandler query = do
sqlGenCtx <- scSQLGenCtx . hcServerCtx <$> ask
GE.explainGQLQuery pgExecCtx sc sqlGenCtx query
v1Alpha1PGDumpHandler :: PGD.PGDumpReqBody -> Handler APIResp
v1Alpha1PGDumpHandler b = do
onlyAdmin
ci <- scConnInfo . hcServerCtx <$> ask
output <- PGD.execPGDump b ci
return $ RawResp "application/sql" output
newtype QueryParser
= QueryParser { getQueryParser :: QualifiedTable -> Handler RQLQuery }
@@ -330,12 +358,12 @@ initErrExit e = do
mkWaiApp
:: Q.TxIsolation -> L.LoggerCtx -> SQLGenCtx
-> Q.PGPool -> HTTP.Manager -> AuthMode
-> Q.PGPool -> Q.ConnInfo -> HTTP.Manager -> AuthMode
-> CorsConfig -> Bool -> Bool
-> InstanceId -> S.HashSet API
-> EL.LQOpts
-> IO (Wai.Application, SchemaCacheRef, Maybe UTCTime)
mkWaiApp isoLevel loggerCtx sqlGenCtx pool httpManager mode corsCfg
mkWaiApp isoLevel loggerCtx sqlGenCtx pool ci httpManager mode corsCfg
enableConsole enableTelemetry instanceId apis
lqOpts = do
let pgExecCtx = PGExecCtx pool isoLevel
@@ -361,7 +389,7 @@ mkWaiApp isoLevel loggerCtx sqlGenCtx pool httpManager mode corsCfg
let schemaCacheRef =
SchemaCacheRef cacheLock cacheRef (E.clearPlanCache planCache)
serverCtx = ServerCtx pgExecCtx logger
serverCtx = ServerCtx pgExecCtx ci logger
schemaCacheRef mode httpManager
sqlGenCtx apis instanceId planCache lqState
@@ -404,36 +432,46 @@ httpApp corsCfg serverCtx enableConsole enableTelemetry = do
put ("v1/template" <//> var) tmpltPutOrPostH
delete ("v1/template" <//> var) tmpltGetOrDeleteH
post "v1/query" $ mkSpockAction encodeQErr serverCtx $ do
post "v1/query" $ mkSpockAction encodeQErr serverCtx $ mkAPIRespHandler $ do
query <- parseBody
v1QueryHandler query
post ("api/1/table" <//> var <//> var) $ \tableName queryType ->
mkSpockAction encodeQErr serverCtx $
mkSpockAction encodeQErr serverCtx $ mkAPIRespHandler $
legacyQueryHandler (TableName tableName) queryType
when enableGraphQL $ do
post "v1alpha1/graphql/explain" $ mkSpockAction encodeQErr serverCtx $ do
expQuery <- parseBody
gqlExplainHandler expQuery
post "v1alpha1/graphql" $ mkSpockAction GH.encodeGQErr serverCtx $ do
when enablePGDump $
post "v1alpha1/pg_dump" $ mkSpockAction encodeQErr serverCtx $ do
query <- parseBody
v1Alpha1GQHandler query
v1Alpha1PGDumpHandler query
when enableGraphQL $ do
post "v1alpha1/graphql/explain" $ mkSpockAction encodeQErr serverCtx $
mkAPIRespHandler $ do
expQuery <- parseBody
gqlExplainHandler expQuery
post "v1alpha1/graphql" $ mkSpockAction GH.encodeGQErr serverCtx $
mkAPIRespHandler $ do
query <- parseBody
v1Alpha1GQHandler query
when (isDeveloperAPIEnabled serverCtx) $ do
get "dev/plan_cache" $ mkSpockAction encodeQErr serverCtx $ do
onlyAdmin
respJ <- liftIO $ E.dumpPlanCache $ scPlanCache serverCtx
return $ encJFromJValue respJ
get "dev/subscriptions" $ mkSpockAction encodeQErr serverCtx $ do
onlyAdmin
respJ <- liftIO $ EL.dumpLiveQueriesState False $ scLQState serverCtx
return $ encJFromJValue respJ
get "dev/subscriptions/extended" $ mkSpockAction encodeQErr serverCtx $ do
onlyAdmin
respJ <- liftIO $ EL.dumpLiveQueriesState True $ scLQState serverCtx
return $ encJFromJValue respJ
get "dev/plan_cache" $ mkSpockAction encodeQErr serverCtx $
mkAPIRespHandler $ do
onlyAdmin
respJ <- liftIO $ E.dumpPlanCache $ scPlanCache serverCtx
return $ encJFromJValue respJ
get "dev/subscriptions" $ mkSpockAction encodeQErr serverCtx $
mkAPIRespHandler $ do
onlyAdmin
respJ <- liftIO $ EL.dumpLiveQueriesState False $ scLQState serverCtx
return $ encJFromJValue respJ
get "dev/subscriptions/extended" $ mkSpockAction encodeQErr serverCtx $
mkAPIRespHandler $ do
onlyAdmin
respJ <- liftIO $ EL.dumpLiveQueriesState True $ scLQState serverCtx
return $ encJFromJValue respJ
forM_ [GET,POST] $ \m -> hookAny m $ \_ -> do
let qErr = err404 NotFound "resource does not exist"
@@ -442,13 +480,15 @@ httpApp corsCfg serverCtx enableConsole enableTelemetry = do
where
enableGraphQL = isGraphQLEnabled serverCtx
enableMetadata = isMetadataEnabled serverCtx
enablePGDump = isPGDumpEnabled serverCtx
tmpltGetOrDeleteH tmpltName = do
tmpltArgs <- tmpltArgsFromQueryParams
mkSpockAction encodeQErr serverCtx $ mkQTemplateAction tmpltName tmpltArgs
mkSpockAction encodeQErr serverCtx $ mkAPIRespHandler $
mkQTemplateAction tmpltName tmpltArgs
tmpltPutOrPostH tmpltName = do
tmpltArgs <- tmpltArgsFromQueryParams
mkSpockAction encodeQErr serverCtx $ do
mkSpockAction encodeQErr serverCtx $ mkAPIRespHandler $ do
bodyTmpltArgs <- parseBody
mkQTemplateAction tmpltName $ M.union bodyTmpltArgs tmpltArgs

View File

@@ -103,6 +103,7 @@ data HGECommandG a
data API
= METADATA
| GRAPHQL
| PGDUMP
| DEVELOPER
deriving (Show, Eq, Read, Generic)
@@ -273,9 +274,9 @@ mkServeOptions rso = do
enableTelemetry strfyNum enabledAPIs lqOpts
where
#ifdef DeveloperAPIs
defaultAPIs = [METADATA,GRAPHQL,DEVELOPER]
defaultAPIs = [METADATA,GRAPHQL,PGDUMP,DEVELOPER]
#else
defaultAPIs = [METADATA,GRAPHQL]
defaultAPIs = [METADATA,GRAPHQL,PGDUMP]
#endif
mkConnParams (RawConnParams s c i p) = do
stripes <- fromMaybe 1 <$> withEnv s (fst pgStripesEnv)
@@ -535,7 +536,7 @@ stringifyNumEnv =
enabledAPIsEnv :: (String, String)
enabledAPIsEnv =
( "HASURA_GRAPHQL_ENABLED_APIS"
, "List of comma separated list of allowed APIs. (default: metadata,graphql)"
, "List of comma separated list of allowed APIs. (default: metadata,graphql,pgdump)"
)
parseRawConnInfo :: Parser RawConnInfo
@@ -693,8 +694,9 @@ readAPIs = mapM readAPI . T.splitOn "," . T.pack
where readAPI si = case T.toUpper $ T.strip si of
"METADATA" -> Right METADATA
"GRAPHQL" -> Right GRAPHQL
"PGDUMP" -> Right PGDUMP
"DEVELOPER" -> Right DEVELOPER
_ -> Left "Only expecting list of comma separated API types metadata / graphql"
_ -> Left "Only expecting list of comma separated API types metadata,graphql,pgdump,developer"
parseWebHook :: Parser RawAuthHook
parseWebHook =

View File

@@ -0,0 +1,67 @@
module Hasura.Server.PGDump
( PGDumpReqBody
, execPGDump
) where
import Control.Exception (IOException, try)
import Data.Aeson.Casing
import Data.Aeson.TH
import qualified Data.ByteString.Lazy as BL
import qualified Data.FileEmbed as FE
import qualified Data.List as L
import qualified Data.Text as T
import qualified Database.PG.Query as Q
import Hasura.Prelude
import qualified Hasura.RQL.Types.Error as RTE
import System.Exit
import System.Process
data PGDumpReqBody =
PGDumpReqBody
{ prbOpts :: ![String]
, prbCleanOutput :: !(Maybe Bool)
} deriving (Show, Eq)
$(deriveJSON (aesonDrop 3 snakeCase) ''PGDumpReqBody)
script :: IsString a => a
script = $(FE.embedStringFile "src-rsr/run_pg_dump.sh")
runScript
:: String
-> [String]
-> String
-> IO (Either String BL.ByteString)
runScript dbUrl opts clean = do
(exitCode, filename, stdErr) <- readProcessWithExitCode "/bin/sh"
["/dev/stdin", dbUrl, unwords opts, clean] script
case exitCode of
ExitSuccess -> do
contents <- BL.readFile $ L.dropWhileEnd (== '\n') filename
return $ Right contents
ExitFailure _ -> return $ Left stdErr
execPGDump
:: (MonadError RTE.QErr m, MonadIO m)
=> PGDumpReqBody
-> Q.ConnInfo
-> m BL.ByteString
execPGDump b ci = do
eOutput <- liftIO $ try $ runScript dbUrl opts clean
output <- either throwException return eOutput
case output of
Left err ->
RTE.throw500 $ "error while executing pg_dump: " <> T.pack err
Right dump -> return dump
where
throwException :: (MonadError RTE.QErr m) => IOException -> m a
throwException _ = RTE.throw500 "internal exception while executing pg_dump"
-- FIXME(shahidhk): need to add connection options (Q.connOptions) too?
dbUrl = "postgres://" <> Q.connUser ci <> ":" <> Q.connPassword ci
<> "@" <> Q.connHost ci <> ":" <> show (Q.connPort ci)
<> "/" <> Q.connDatabase ci
opts = prbOpts b
clean = case prbCleanOutput b of
Just v -> show v
Nothing -> show False

View File

@@ -0,0 +1,47 @@
#! /usr/bin/env sh
set -e
filename=/tmp/pg_dump-$(date +%s).sql
template_file=/tmp/hasura_del_lines_template.txt
# input args
DB_URL=$1
OPTS=$2
CLEAN=$3
pg_dump "$DB_URL" $OPTS -f "$filename"
# clean the file the variable is True
if [ "$CLEAN" = "True" ]; then
# delete all comments
sed -i '/^--/d' "$filename"
# delete front matter
cat > $template_file << EOF
SET statement_timeout = 0;
SET lock_timeout = 0;
SET idle_in_transaction_session_timeout = 0;
SET client_encoding = 'UTF8';
SET standard_conforming_strings = on;
SELECT pg_catalog.set_config('search_path', '', false);
SET check_function_bodies = false;
SET client_min_messages = warning;
SET row_security = off;
SET default_tablespace = '';
SET default_with_oids = false;
CREATE SCHEMA public;
COMMENT ON SCHEMA public IS 'standard public schema';
EOF
while read -r line; do
sed -i '/^'"$line"'$/d' "$filename"
done < $template_file
# delete notify triggers
sed -i -E '/^CREATE TRIGGER "?notify_hasura_.+"? AFTER \w+ ON .+ FOR EACH ROW EXECUTE PROCEDURE "?hdb_views"?\."?notify_hasura_.+"?\(\);$/d' "$filename"
# delete empty lines
sed -i '/^[[:space:]]*$/d' "$filename"
fi
printf "%s" "$filename"

View File

@@ -0,0 +1,81 @@
descriptions: Execute pg_dump on public schema
url: /v1alpha1/pg_dump
status: 200
query:
opts:
- -O
- -x
- --schema-only
- --schema
- public
clean_output: true
# response on postgres 9.4 and 9.5
response_9: |
CREATE TABLE public.articles (
id integer NOT NULL,
author_id integer NOT NULL,
title text NOT NULL,
body text NOT NULL
);
CREATE SEQUENCE public.articles_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
ALTER SEQUENCE public.articles_id_seq OWNED BY public.articles.id;
CREATE TABLE public.authors (
id integer NOT NULL,
name text NOT NULL
);
CREATE SEQUENCE public.authors_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
ALTER SEQUENCE public.authors_id_seq OWNED BY public.authors.id;
ALTER TABLE ONLY public.articles ALTER COLUMN id SET DEFAULT nextval('public.articles_id_seq'::regclass);
ALTER TABLE ONLY public.authors ALTER COLUMN id SET DEFAULT nextval('public.authors_id_seq'::regclass);
ALTER TABLE ONLY public.articles
ADD CONSTRAINT articles_pkey PRIMARY KEY (id);
ALTER TABLE ONLY public.authors
ADD CONSTRAINT authors_pkey PRIMARY KEY (id);
ALTER TABLE ONLY public.articles
ADD CONSTRAINT articles_author_id_fkey FOREIGN KEY (author_id) REFERENCES public.authors(id);
# response on postgres 10 and 11
response_10_11: |
CREATE TABLE public.articles (
id integer NOT NULL,
author_id integer NOT NULL,
title text NOT NULL,
body text NOT NULL
);
CREATE SEQUENCE public.articles_id_seq
AS integer
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
ALTER SEQUENCE public.articles_id_seq OWNED BY public.articles.id;
CREATE TABLE public.authors (
id integer NOT NULL,
name text NOT NULL
);
CREATE SEQUENCE public.authors_id_seq
AS integer
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
ALTER SEQUENCE public.authors_id_seq OWNED BY public.authors.id;
ALTER TABLE ONLY public.articles ALTER COLUMN id SET DEFAULT nextval('public.articles_id_seq'::regclass);
ALTER TABLE ONLY public.authors ALTER COLUMN id SET DEFAULT nextval('public.authors_id_seq'::regclass);
ALTER TABLE ONLY public.articles
ADD CONSTRAINT articles_pkey PRIMARY KEY (id);
ALTER TABLE ONLY public.authors
ADD CONSTRAINT authors_pkey PRIMARY KEY (id);
ALTER TABLE ONLY public.articles
ADD CONSTRAINT articles_author_id_fkey FOREIGN KEY (author_id) REFERENCES public.authors(id);

View File

@@ -0,0 +1,45 @@
type: bulk
args:
- type: run_sql
args:
sql: |
CREATE TABLE public.authors (
id serial NOT NULL PRIMARY KEY,
name text NOT NULL
);
CREATE TABLE public.articles (
id serial NOT NULL PRIMARY KEY,
author_id integer NOT NULL REFERENCES public.authors(id),
title text NOT NULL,
body text NOT NULL
);
- args:
name: authors
schema: public
type: track_table
- args:
name: articles
schema: public
type: track_table
- args:
delete:
columns: '*'
headers: []
insert:
columns: '*'
name: articles
retry_conf:
interval_sec: 10
num_retries: 0
timeout_sec: 60
table:
name: articles
schema: public
update:
columns:
- author_id
- body
- id
- title
webhook: https://httpbin.org/post
type: create_event_trigger

View File

@@ -0,0 +1,8 @@
type: bulk
args:
- args:
cascade: true
sql: |
DROP TABLE articles;
DROP TABLE authors;
type: run_sql

View File

@@ -0,0 +1,30 @@
import yaml
from super_classes import DefaultTestSelectQueries
import os
resp_pg_version_map = {
'9_5': 'response_9',
'9_6': 'response_9',
'10_6': 'response_10_11',
'11_1': 'response_10_11',
'latest': 'response_10_11'
}
class TestPGDump(DefaultTestSelectQueries):
def test_pg_dump_for_public_schema(self, hge_ctx):
query_file = self.dir() + '/pg_dump_public.yaml'
PG_VERSION = os.getenv('PG_VERSION', 'latest')
with open(query_file, 'r') as stream:
q = yaml.safe_load(stream)
headers = {}
if hge_ctx.hge_key is not None:
headers['x-hasura-admin-secret'] = hge_ctx.hge_key
resp = hge_ctx.http.post(hge_ctx.hge_url + q['url'], json=q['query'], headers=headers)
body = resp.text
assert resp.status_code == q['status']
assert body == q[resp_pg_version_map[PG_VERSION]]
@classmethod
def dir(cls):
return "pgdump"