splitgraph.cloud package

Subpackages

Submodules

splitgraph.cloud.models module

Definitions for responses from the cloud GQL/REST APIs

class splitgraph.cloud.models.AddExternalCredentialRequest(*, credential_name: str, credential_data: Dict[str, Any], plugin_name: str)

Bases: pydantic.main.BaseModel

credential_data: Dict[str, Any]

credential_name: str

plugin_name: str

class splitgraph.cloud.models.AddExternalRepositoriesRequest(*, repositories: List[splitgraph.cloud.models.AddExternalRepositoryRequest])

Bases: pydantic.main.BaseModel

repositories: List[splitgraph.cloud.models.AddExternalRepositoryRequest]

class splitgraph.cloud.models.AddExternalRepositoryRequest(*, namespace: str, repository: str, plugin_name: str, params: Params, is_live: bool, tables: Dict[str, splitgraph.cloud.models.ExternalTableRequest] = None, credential_id: str = None, credential_name: str = None, credential_data: Dict[str, Any] = None, schedule: splitgraph.cloud.project.models.IngestionSchedule = None, initial_private: bool = False)

Bases: pydantic.main.BaseModel

credential_data: Optional[Dict[str, Any]]

credential_id: Optional[str]

credential_name: Optional[str]

classmethod from_external(namespace: str, repository: str, external: splitgraph.cloud.project.models.External, credential_map: Optional[Dict[str, str]] = None, initial_private: bool = False)

initial_private: bool

is_live: bool

namespace: str

params: Params

plugin_name: str

repository: str

schedule: Optional[splitgraph.cloud.project.models.IngestionSchedule]

tables: Optional[Dict[str, splitgraph.cloud.models.ExternalTableRequest]]

class splitgraph.cloud.models.ExportJobStatus(*, task_id: str, started: datetime.datetime, finished: datetime.datetime = None, status: str = None, user_id: str = None, export_format: str, output: Dict[str, Any] = None)

Bases: splitgraph.cloud.models.JobStatus

export_format: str

output: Optional[Dict[str, Any]]

user_id: Optional[str]

class splitgraph.cloud.models.ExternalResponse(*, namespace: str, repository: str, credentialId: str = None, dataSource: str, params: Dict[str, Any], tableParams: Dict[str, Any], externalImageByNamespaceAndRepository: splitgraph.cloud.models.ExternalResponse.ExternalImageResponse = None, ingestionScheduleByNamespaceAndRepository: splitgraph.cloud.models.ExternalResponse.IngestionScheduleResponse = None)

Bases: pydantic.main.BaseModel

class ExternalImageResponse(*, imageByNamespaceAndRepositoryAndImageHash: splitgraph.cloud.models.ExternalResponse.ExternalImageResponse.ImageResponse)

Bases: pydantic.main.BaseModel

class ImageResponse(*, tablesByNamespaceAndRepositoryAndImageHash: splitgraph.cloud.models.ExternalResponse.ExternalImageResponse.ImageResponse.TablesResponse)

Bases: pydantic.main.BaseModel

class TablesResponse(*, nodes: List[splitgraph.cloud.models.ExternalResponse.ExternalImageResponse.ImageResponse.TablesResponse.Node])

Bases: pydantic.main.BaseModel

class Node(*, tableName: str, tableSchema: List[splitgraph.core.types.TableColumn])

Bases: pydantic.main.BaseModel

tableName: str

tableSchema: List[splitgraph.core.types.TableColumn]

nodes: List[splitgraph.cloud.models.ExternalResponse.ExternalImageResponse.ImageResponse.TablesResponse.Node]

tablesByNamespaceAndRepositoryAndImageHash: splitgraph.cloud.models.ExternalResponse.ExternalImageResponse.ImageResponse.TablesResponse

imageByNamespaceAndRepositoryAndImageHash: splitgraph.cloud.models.ExternalResponse.ExternalImageResponse.ImageResponse

class IngestionScheduleResponse(*, schedule: str, schema: Dict[str, List[splitgraph.core.types.TableColumn]], enabled: bool = True)

Bases: pydantic.main.BaseModel

schedule: str

schema_: Dict[str, List[splitgraph.core.types.TableColumn]]

credentialId: Optional[str]

dataSource: str

externalImageByNamespaceAndRepository: Optional[splitgraph.cloud.models.ExternalResponse.ExternalImageResponse]

classmethod from_response(response) → List[splitgraph.cloud.models.ExternalResponse]

ingestionScheduleByNamespaceAndRepository: Optional[splitgraph.cloud.models.ExternalResponse.IngestionScheduleResponse]

namespace: str

params: Dict[str, Any]

repository: str

tableParams: Dict[str, Any]

to_external() → splitgraph.cloud.project.models.External

class splitgraph.cloud.models.ExternalTableRequest(*, options: Dict[str, Any] = {}, schema: Dict[str, str] = None)

Bases: pydantic.main.BaseModel

options: Dict[str, Any]

schema_: Optional[Dict[str, str]]

class splitgraph.cloud.models.IngestionJobStatus(*, task_id: str, started: datetime.datetime, finished: datetime.datetime = None, status: str = None, is_manual: bool)

Bases: splitgraph.cloud.models.JobStatus

is_manual: bool

class splitgraph.cloud.models.JobStatus(*, task_id: str, started: datetime.datetime, finished: datetime.datetime = None, status: str = None)

Bases: pydantic.main.BaseModel

finished: Optional[datetime.datetime]

started: datetime.datetime

status: Optional[str]

task_id: str

class splitgraph.cloud.models.ListExternalCredentialsResponse(*, credentials: List[splitgraph.cloud.models.ListExternalCredentialsResponse.ExternalCredential])

Bases: pydantic.main.BaseModel

class ExternalCredential(*, plugin_name: str, credential_name: str = None, credential_id: str)

Bases: pydantic.main.BaseModel

credential_id: str

credential_name: Optional[str]

plugin_name: str

credentials: List[splitgraph.cloud.models.ListExternalCredentialsResponse.ExternalCredential]

class splitgraph.cloud.models.MetadataResponse(*, namespace: str, repository: str, repoTopicsByNamespaceAndRepository: splitgraph.cloud.models.MetadataResponse.RepoTopicsResponse, repoProfileByNamespaceAndRepository: splitgraph.cloud.models.MetadataResponse.RepoProfileResponse)

Bases: pydantic.main.BaseModel

class RepoProfileResponse(*, readme: str = None, description: str = None, topics: List[str] = None, sources: List[splitgraph.cloud.project.models.Source] = None, license: str = None, metadata: splitgraph.cloud.models.MetadataResponse.RepoProfileResponse.MetadataResponse = None)

Bases: pydantic.main.BaseModel

class MetadataResponse(*, created_at: str = None, updated_at: str = None, upstream_metadata: Dict[str, Any])

Bases: pydantic.main.BaseModel

created_at: Optional[str]

updated_at: Optional[str]

upstream_metadata: Dict[str, Any]

description: Optional[str]

license: Optional[str]

metadata: Optional[splitgraph.cloud.models.MetadataResponse.RepoProfileResponse.MetadataResponse]

readme: Optional[str]

sources: Optional[List[splitgraph.cloud.project.models.Source]]

topics: Optional[List[str]]

class RepoTopicsResponse(*, nodes: List[splitgraph.cloud.models.MetadataResponse.RepoTopicsResponse.RepositoryTopics])

Bases: pydantic.main.BaseModel

class RepositoryTopics(*, topic: str)

Bases: pydantic.main.BaseModel

topic: str

nodes: List[splitgraph.cloud.models.MetadataResponse.RepoTopicsResponse.RepositoryTopics]

classmethod from_response(response) → List[splitgraph.cloud.models.MetadataResponse]

namespace: str

repoProfileByNamespaceAndRepository: splitgraph.cloud.models.MetadataResponse.RepoProfileResponse

repoTopicsByNamespaceAndRepository: splitgraph.cloud.models.MetadataResponse.RepoTopicsResponse

repository: str

to_metadata() → splitgraph.cloud.project.models.Metadata

class splitgraph.cloud.models.Plugin(*, plugin_name: str, credentials_schema: Dict[str, Any], params_schema: Dict[str, Any], table_params_schema: Dict[str, Any], name: str, description: str, supports_mount: bool, supports_load: bool, supports_sync: bool)

Bases: pydantic.main.BaseModel

credentials_schema: Dict[str, Any]

description: str

name: str

params_schema: Dict[str, Any]

plugin_name: str

supports_load: bool

supports_mount: bool

supports_sync: bool

table_params_schema: Dict[str, Any]

class splitgraph.cloud.models.RepositoryIngestionJobStatusResponse(*, repositoryIngestionJobStatus: splitgraph.cloud.models.RepositoryIngestionJobStatusResponse.RepositoryIngestionJobStatus)

Bases: pydantic.main.BaseModel

class RepositoryIngestionJobStatus(*, nodes: List[splitgraph.cloud.models.RepositoryIngestionJobStatusResponse.RepositoryIngestionJobStatus.Node])

Bases: pydantic.main.BaseModel

class Node(*, taskId: str, started: datetime.datetime, finished: datetime.datetime = None, isManual: bool, status: str = None)

Bases: pydantic.main.BaseModel

finished: Optional[datetime.datetime]

isManual: bool

started: datetime.datetime

status: Optional[str]

taskId: str

nodes: List[splitgraph.cloud.models.RepositoryIngestionJobStatusResponse.RepositoryIngestionJobStatus.Node]

classmethod from_response(response) → splitgraph.cloud.models.RepositoryIngestionJobStatusResponse

repositoryIngestionJobStatus: splitgraph.cloud.models.RepositoryIngestionJobStatusResponse.RepositoryIngestionJobStatus

class splitgraph.cloud.models.UpdateExternalCredentialRequest(*, credential_id: str, credential_name: str, credential_data: Dict[str, Any], plugin_name: str)

Bases: pydantic.main.BaseModel

credential_data: Dict[str, Any]

credential_id: str

credential_name: str

plugin_name: str

class splitgraph.cloud.models.UpdateExternalCredentialResponse(*, credential_id: str)

Bases: pydantic.main.BaseModel

credential_id: str

splitgraph.cloud.models.make_repositories(metadata_responses: List[splitgraph.cloud.models.MetadataResponse], external_responses: List[splitgraph.cloud.models.ExternalResponse]) → List[splitgraph.cloud.project.models.Repository]

splitgraph.cloud.queries module

Module contents

Public API for interacting with the Splitgraph registry

splitgraph.cloud.AuthAPIClient(*args, **kwargs)

class splitgraph.cloud.GQLAPIClient(remote: Optional[str], endpoint: Optional[str] = None, access_token: Optional[str] = None)

Bases: object

Wrapper class for select Splitgraph Registry GQL operations that can be called from the CLI

property access_token: Optional[str]

bulk_upsert_metadata(namespace_list: List[str], repository_list: List[str], metadata_list: List[splitgraph.cloud.project.models.Metadata])

find_repository(query: str, limit: int = 10) → Tuple[int, List[Tuple[str, str, str]]]

get_all_plugins() → List[splitgraph.cloud.models.Plugin]

get_csv_upload_download_urls() → Tuple[str, str]

get_export_job_status(task_id: str) → Optional[splitgraph.cloud.models.ExportJobStatus]

get_external_metadata(namespace: str, repository: str) → Optional[splitgraph.cloud.models.ExternalResponse]

get_ingestion_job_logs(namespace: str, repository: str, task_id: str) → str

get_latest_ingestion_job_status(namespace: str, repository: str) → Optional[splitgraph.cloud.models.IngestionJobStatus]

get_metadata(namespace: str, repository: str) → Optional[splitgraph.cloud.models.MetadataResponse]

get_plugin(plugin_name: str) → Optional[splitgraph.cloud.models.Plugin]

load_all_repositories(limit_to: Optional[List[str]] = None) → List[splitgraph.cloud.project.models.Repository]

property maybe_access_token: Optional[str]

start_csv_load(namespace: str, repository: str, download_urls: List[str], table_names: List[str]) → str

start_export(query: str) → str

start_load_existing(namespace: str, repository: str, sync: bool = True) → str

start_load_params(namespace: str, repository: str, external: splitgraph.cloud.project.models.External, sync: bool = True, credential_data: Optional[Dict[str, Any]] = None, initial_private: bool = False) → str

upsert_description(namespace: str, repository: str, description: str)

upsert_metadata(namespace: str, repository: str, metadata: splitgraph.cloud.project.models.Metadata)

Update metadata for a single repository.

upsert_readme(namespace: str, repository: str, readme: str)

upsert_topics(namespace: str, repository: str, topics: List[str])

class splitgraph.cloud.RESTAPIClient(remote: str)

Bases: object

Client for various Splitgraph Registry REST APIs: auth token generation, external repo setup…

property access_token: str

Will return an up-to-date access token by either getting it from the configuration file or contacting the auth service for a new one. Will write the new access token into the configuration file.

Returns

Access token.

bulk_upsert_external(repositories: List[splitgraph.cloud.models.AddExternalRepositoryRequest])

create_machine_credentials(access_token: str, password: str) → requests.models.Response

Generate a key and secret that can be used to log into the Splitgraph registry via a normal Postgres connection. The secret must be stored in the user’s local configuration file (it’s not stored on Splitgraph servers).

Parameters

• access_token – Access token

• password – Password

Returns

Tuple of (key, secret).

ensure_external_credential(credential_data: Dict[str, Any], credential_name: str, plugin_name: str) → str

Store a credential for accessing an external data source in Splitgraph Cloud and return its ID

get_access_token(refresh_token: str) → requests.models.Response

Get a new access token from a refresh token.

Parameters

refresh_token – Refresh token

Returns

New access token.

get_access_token_from_api(api_key: str, api_secret: str) → requests.models.Response

Get a new access token from API keys

Parameters

• api_key – API key

• api_secret – API secret

Returns

New access token.

get_latest_version() → Optional[str]

get_refresh_token(username: str, password: str) → requests.models.Response

Get a long-lived refresh token and a short-lived access token from the API.

Parameters

• username – Username

• password – Password

Returns

Tuple of (access_token, refresh_token).

list_external_credentials() → splitgraph.cloud.models.ListExternalCredentialsResponse

property maybe_access_token: Optional[str]

Like access_token but returns None if the user isn’t logged in.

register(username: str, password: str, email: str, accept_tos: bool) → requests.models.Response

Register a new Splitgraph user.

Parameters

• username – Username

• password – Password

• email – Email

• accept_tos – Accept the Terms of Service if they exist

tos() → requests.models.Response

Get a Terms of Service message from the registry (if accepting ToS is required) :return: Link to the Terms of Service or None

splitgraph.cloud.expect_result(result: List[str], ignore_status_codes: Optional[List[int]] = None) → Callable[[Callable[[...], requests.models.Response]], Callable[[...], Union[str, Tuple[str]]]]

A decorator that can be wrapped around a function returning a requests.Response with a JSON body. If the request has failed, it will extract the “error” from the JSON response and raise an AuthAPIError.

Parameters

• result – Items to extract. Will raise an AuthAPIError if not all items were fetched.

• ignore_status_codes – If one of these status codes is returned (e.g. 404), it gets ignored and a None is returned instead.

Returns

Tuple of items enumerated in the result list. If there’s only one item, it will return just that item.

splitgraph.cloud.get_headers()

splitgraph.cloud.get_remote_param(remote: str, key: str) → str

splitgraph.cloud.get_token_claim(jwt, claim)

Extract a claim from a JWT token without validating it.

splitgraph.cloud.handle_gql_errors(func: Callable[[...], requests.models.Response]) → Callable[[...], requests.models.Response]

A decorator that handles responses from the GQL API, transforming errors into exceptions.