Skip to content

Commit 1f46be3

Browse files
epikhinmepikhinm
committed
docs: rearrange examples to dedicated files
1 parent 31b47db commit 1f46be3

22 files changed

+316
-288
lines changed

Diff for: ‎examples/clickhouse/create.py

+29
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
import logging
2+
3+
from google.protobuf.wrappers_pb2 import Int64Value
4+
5+
from doublecloud.clickhouse.v1.cluster_pb2 import ClusterResources
6+
from doublecloud.clickhouse.v1.cluster_service_pb2 import CreateClusterRequest
7+
from doublecloud.clickhouse.v1.cluster_service_pb2_grpc import ClusterServiceStub
8+
9+
10+
def create_cluster(sdk, project_id, region_id, name, network_id):
11+
cluster_service = sdk.client(ClusterServiceStub)
12+
operation = cluster_service.Create(
13+
CreateClusterRequest(
14+
project_id=project_id,
15+
cloud_type="aws",
16+
region_id=region_id,
17+
name=name,
18+
resources=ClusterResources(
19+
clickhouse=ClusterResources.Clickhouse(
20+
resource_preset_id="s1-c2-m4",
21+
disk_size=Int64Value(value=32 * 2**30),
22+
replica_count=Int64Value(value=1),
23+
)
24+
),
25+
network_id=network_id,
26+
)
27+
)
28+
logging.info("Creating initiated")
29+
return operation

Diff for: ‎examples/clickhouse/delete.py

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
from doublecloud.clickhouse.v1.cluster_service_pb2 import DeleteClusterRequest
2+
from doublecloud.clickhouse.v1.cluster_service_pb2_grpc import ClusterServiceStub
3+
4+
5+
def delete_cluster(sdk, cluster_id):
6+
cluster_service = sdk.client(ClusterServiceStub)
7+
return cluster_service.Delete(DeleteClusterRequest(cluster_id=cluster_id))

Diff for: ‎examples/clickhouse/main.py

+3-34
Original file line numberDiff line numberDiff line change
@@ -3,42 +3,11 @@
33
import json
44
import logging
55

6-
from google.protobuf.wrappers_pb2 import Int64Value
6+
# pylint: disable=E0401
7+
from create import create_cluster
8+
from delete import delete_cluster
79

810
import doublecloud
9-
from doublecloud.clickhouse.v1.cluster_pb2 import ClusterResources
10-
from doublecloud.clickhouse.v1.cluster_service_pb2 import (
11-
CreateClusterRequest,
12-
DeleteClusterRequest,
13-
)
14-
from doublecloud.clickhouse.v1.cluster_service_pb2_grpc import ClusterServiceStub
15-
16-
17-
def create_cluster(sdk, project_id, region_id, name, network_id):
18-
cluster_service = sdk.client(ClusterServiceStub)
19-
operation = cluster_service.Create(
20-
CreateClusterRequest(
21-
project_id=project_id,
22-
cloud_type="aws",
23-
region_id=region_id,
24-
name=name,
25-
resources=ClusterResources(
26-
clickhouse=ClusterResources.Clickhouse(
27-
resource_preset_id="s1-c2-m4",
28-
disk_size=Int64Value(value=32 * 2**30),
29-
replica_count=Int64Value(value=1),
30-
)
31-
),
32-
network_id=network_id,
33-
)
34-
)
35-
logging.info("Creating initiated")
36-
return operation
37-
38-
39-
def delete_cluster(sdk, cluster_id):
40-
cluster_service = sdk.client(ClusterServiceStub)
41-
return cluster_service.Delete(DeleteClusterRequest(cluster_id=cluster_id))
4211

4312

4413
def main():

Diff for: ‎examples/kafka/create.py

+30
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
import logging
2+
3+
from google.protobuf.wrappers_pb2 import Int64Value
4+
5+
from doublecloud.kafka.v1.cluster_pb2 import ClusterResources
6+
from doublecloud.kafka.v1.cluster_service_pb2 import CreateClusterRequest
7+
from doublecloud.kafka.v1.cluster_service_pb2_grpc import ClusterServiceStub
8+
9+
10+
def create_cluster(sdk, project_id, region_id, name, network_id):
11+
cluster_service = sdk.client(ClusterServiceStub)
12+
operation = cluster_service.Create(
13+
CreateClusterRequest(
14+
project_id=project_id,
15+
cloud_type="aws",
16+
region_id=region_id,
17+
name=name,
18+
resources=ClusterResources(
19+
kafka=ClusterResources.Kafka(
20+
resource_preset_id="s1-c2-m4",
21+
disk_size=Int64Value(value=32 * 2**30),
22+
broker_count=Int64Value(value=1),
23+
zone_count=Int64Value(value=1),
24+
)
25+
),
26+
network_id=network_id,
27+
)
28+
)
29+
logging.info("Creating initiated")
30+
return operation

Diff for: ‎examples/kafka/delete.py

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
from doublecloud.kafka.v1.cluster_service_pb2 import DeleteClusterRequest
2+
from doublecloud.kafka.v1.cluster_service_pb2_grpc import ClusterServiceStub
3+
4+
5+
def delete_cluster(sdk, cluster_id):
6+
cluster_service = sdk.client(ClusterServiceStub)
7+
return cluster_service.Delete(DeleteClusterRequest(cluster_id=cluster_id))

Diff for: ‎examples/kafka/main.py

+3-35
Original file line numberDiff line numberDiff line change
@@ -3,43 +3,11 @@
33
import json
44
import logging
55

6-
from google.protobuf.wrappers_pb2 import Int64Value
6+
# pylint: disable=E0401
7+
from create import create_cluster
8+
from delete import delete_cluster
79

810
import doublecloud
9-
from doublecloud.kafka.v1.cluster_pb2 import ClusterResources
10-
from doublecloud.kafka.v1.cluster_service_pb2 import (
11-
CreateClusterRequest,
12-
DeleteClusterRequest,
13-
)
14-
from doublecloud.kafka.v1.cluster_service_pb2_grpc import ClusterServiceStub
15-
16-
17-
def create_cluster(sdk, project_id, region_id, name, network_id):
18-
cluster_service = sdk.client(ClusterServiceStub)
19-
operation = cluster_service.Create(
20-
CreateClusterRequest(
21-
project_id=project_id,
22-
cloud_type="aws",
23-
region_id=region_id,
24-
name=name,
25-
resources=ClusterResources(
26-
kafka=ClusterResources.Kafka(
27-
resource_preset_id="s1-c2-m4",
28-
disk_size=Int64Value(value=32 * 2**30),
29-
broker_count=Int64Value(value=1),
30-
zone_count=Int64Value(value=1),
31-
)
32-
),
33-
network_id=network_id,
34-
)
35-
)
36-
logging.info("Creating initiated")
37-
return operation
38-
39-
40-
def delete_cluster(sdk, cluster_id):
41-
cluster_service = sdk.client(ClusterServiceStub)
42-
return cluster_service.Delete(DeleteClusterRequest(cluster_id=cluster_id))
4311

4412

4513
def main():

Diff for: ‎examples/transfer/activate.py

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
from doublecloud.transfer.v1.transfer_service_pb2 import ActivateTransferRequest
2+
3+
4+
def activate_transfer(svc, transfer_id: str):
5+
return svc.Activate(ActivateTransferRequest(transfer_id=transfer_id))

Diff for: ‎examples/transfer/create.py

+10
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
from doublecloud.transfer.v1.transfer_pb2 import TransferType
2+
from doublecloud.transfer.v1.transfer_service_pb2 import CreateTransferRequest
3+
4+
5+
def create_transfer(svc, project_id: str, name: str, src_id: str, dst_id: str):
6+
return svc.Create(
7+
CreateTransferRequest(
8+
source_id=src_id, target_id=dst_id, name=name, project_id=project_id, type=TransferType.SNAPSHOT_ONLY
9+
)
10+
)

Diff for: ‎examples/transfer/deactivate.py

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
from doublecloud.transfer.v1.transfer_service_pb2 import DeactivateTransferRequest
2+
3+
4+
def deactivate_transfer(svc, transfer_id: str):
5+
return svc.Deactivate(DeactivateTransferRequest(transfer_id=transfer_id))

Diff for: ‎examples/transfer/delete.py

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
from doublecloud.transfer.v1.transfer_service_pb2 import DeleteTransferRequest
2+
3+
4+
def delete_transfer(svc, transfer_id: str):
5+
return svc.Delete(DeleteTransferRequest(transfer_id=transfer_id))

Diff for: ‎examples/transfer/endpoints.py

+55
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
from doublecloud.transfer.v1.endpoint.airbyte.s3_source_pb2 import S3Source
2+
from doublecloud.transfer.v1.endpoint.clickhouse_pb2 import (
3+
ClickhouseConnection,
4+
ClickhouseConnectionOptions,
5+
ClickhouseTarget,
6+
)
7+
from doublecloud.transfer.v1.endpoint.common_pb2 import Secret
8+
from doublecloud.transfer.v1.endpoint_pb2 import EndpointSettings
9+
from doublecloud.transfer.v1.endpoint_service_pb2 import (
10+
CreateEndpointRequest,
11+
DeleteEndpointRequest,
12+
)
13+
14+
15+
def create_s3_src_endpoint(svc, project_id: str, name: str):
16+
return svc.Create(
17+
CreateEndpointRequest(
18+
project_id=project_id,
19+
name=f"s3-src-{name}",
20+
settings=EndpointSettings(
21+
s3_source=S3Source(
22+
dataset="test",
23+
path_pattern="test",
24+
schema="test",
25+
format=S3Source.Format(csv=S3Source.Csv()),
26+
provider=S3Source.Provider(bucket="test"),
27+
)
28+
),
29+
)
30+
)
31+
32+
33+
def create_ch_dst_endpoint(svc, project_id: str, name: str):
34+
return svc.Create(
35+
CreateEndpointRequest(
36+
project_id=project_id,
37+
name=f"ch-dst-{name}",
38+
settings=EndpointSettings(
39+
clickhouse_target=ClickhouseTarget(
40+
connection=ClickhouseConnection(
41+
connection_options=ClickhouseConnectionOptions(
42+
mdb_cluster_id="xoxo",
43+
database="default",
44+
user="user",
45+
password=Secret(raw="98s*%^P!3Bw38"),
46+
)
47+
)
48+
)
49+
),
50+
)
51+
)
52+
53+
54+
def delete_endpoint(svc, endpoint_id: str):
55+
return svc.Delete(DeleteEndpointRequest(endpoint_id=endpoint_id))

Diff for: ‎examples/transfer/main.py

+7-82
Original file line numberDiff line numberDiff line change
@@ -3,93 +3,18 @@
33
import json
44
import logging
55

6+
# pylint: disable=E0401
7+
from activate import activate_transfer
8+
from create import create_transfer
9+
from deactivate import deactivate_transfer
10+
from delete import delete_transfer
11+
from endpoints import create_ch_dst_endpoint, create_s3_src_endpoint, delete_endpoint
12+
613
import doublecloud
7-
from doublecloud.transfer.v1.endpoint.airbyte.s3_source_pb2 import S3Source
8-
from doublecloud.transfer.v1.endpoint.clickhouse_pb2 import (
9-
ClickhouseConnection,
10-
ClickhouseConnectionOptions,
11-
ClickhouseTarget,
12-
)
13-
from doublecloud.transfer.v1.endpoint.common_pb2 import Secret
14-
from doublecloud.transfer.v1.endpoint_pb2 import EndpointSettings
15-
from doublecloud.transfer.v1.endpoint_service_pb2 import (
16-
CreateEndpointRequest,
17-
DeleteEndpointRequest,
18-
)
1914
from doublecloud.transfer.v1.endpoint_service_pb2_grpc import EndpointServiceStub
20-
from doublecloud.transfer.v1.transfer_pb2 import TransferType
21-
from doublecloud.transfer.v1.transfer_service_pb2 import (
22-
ActivateTransferRequest,
23-
CreateTransferRequest,
24-
DeactivateTransferRequest,
25-
DeleteTransferRequest,
26-
)
2715
from doublecloud.transfer.v1.transfer_service_pb2_grpc import TransferServiceStub
2816

2917

30-
def create_s3_src_endpoint(svc, project_id: str, name: str):
31-
return svc.Create(
32-
CreateEndpointRequest(
33-
project_id=project_id,
34-
name=f"s3-src-{name}",
35-
settings=EndpointSettings(
36-
s3_source=S3Source(
37-
dataset="test",
38-
path_pattern="test",
39-
schema="test",
40-
format=S3Source.Format(csv=S3Source.Csv()),
41-
provider=S3Source.Provider(bucket="test"),
42-
)
43-
),
44-
)
45-
)
46-
47-
48-
def delete_endpoint(svc, endpoint_id: str):
49-
return svc.Delete(DeleteEndpointRequest(endpoint_id=endpoint_id))
50-
51-
52-
def create_ch_dst_endpoint(svc, project_id: str, name: str):
53-
return svc.Create(
54-
CreateEndpointRequest(
55-
project_id=project_id,
56-
name=f"ch-dst-{name}",
57-
settings=EndpointSettings(
58-
clickhouse_target=ClickhouseTarget(
59-
connection=ClickhouseConnection(
60-
connection_options=ClickhouseConnectionOptions(
61-
mdb_cluster_id="xoxo",
62-
database="default",
63-
user="user",
64-
password=Secret(raw="98s*%^P!3Bw38"),
65-
)
66-
)
67-
)
68-
),
69-
)
70-
)
71-
72-
73-
def create_transfer(svc, project_id: str, name: str, src_id: str, dst_id: str):
74-
return svc.Create(
75-
CreateTransferRequest(
76-
source_id=src_id, target_id=dst_id, name=name, project_id=project_id, type=TransferType.SNAPSHOT_ONLY
77-
)
78-
)
79-
80-
81-
def activate_transfer(svc, transfer_id: str):
82-
return svc.Activate(ActivateTransferRequest(transfer_id=transfer_id))
83-
84-
85-
def deactivate_transfer(svc, transfer_id: str):
86-
return svc.Deactivate(DeactivateTransferRequest(transfer_id=transfer_id))
87-
88-
89-
def delete_transfer(svc, transfer_id: str):
90-
return svc.Delete(DeleteTransferRequest(transfer_id=transfer_id))
91-
92-
9318
def main():
9419
logging.basicConfig(level=logging.INFO)
9520
arguments = parse_args()

Diff for: ‎examples/visualization/advise_dataset_fields.py

+27
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
from doublecloud.visualization.v1.workbook_pb2 import Dataset
2+
from doublecloud.visualization.v1.workbook_service_pb2 import AdviseDatasetFieldsRequest
3+
4+
5+
def advise_dataset_fields(svc, workbook_id: str, sources: list, connection_name: str):
6+
"""
7+
Function helps to define automatically all fields, their names/IDs and types
8+
based on underlying datasource (table, view, SQL query, etc.).
9+
ID of fields will be equals to column names.
10+
You can use define them manually or use this handler to simplifying for popular cases
11+
"""
12+
dataset = Dataset()
13+
dataset.config.struct_value.update(
14+
{
15+
"fields": [],
16+
"avatars": None,
17+
"sources": sources,
18+
}
19+
)
20+
21+
return svc.AdviseDatasetFields(
22+
AdviseDatasetFieldsRequest(
23+
workbook_id=workbook_id,
24+
connection_name=connection_name,
25+
partial_dataset=dataset,
26+
)
27+
)

Diff for: ‎examples/visualization/create.py

+10
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
from doublecloud.v1.operation_pb2 import Operation
2+
from doublecloud.visualization.v1.workbook_service_pb2 import CreateWorkbookRequest
3+
4+
5+
def create_workbook(svc, project_id: str, name: str) -> Operation:
6+
"""
7+
Function creates an empty workbook
8+
We will fill it with other functions
9+
"""
10+
return svc.Create(CreateWorkbookRequest(project_id=project_id, workbook_title=name))

0 commit comments

Comments
 (0)