target-bigquery/meltano.yml at main · z3z1ma/target-bigquery · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
version: 1
send_anonymous_usage_stats: true
project_id: target-bigquery
default_environment: testing
environments:
- name: testing
plugins:
  extractors:
  - name: tap-infinity
    namespace: tap_infinity
    pip_url: git+https://github.com/spacecowboy/tap-infinity.git
    capabilities:
    - catalog
    - discover
    - batch
    - test
    - about
    config:
      row_count: 1000000
      column_count: 150
      batch_size: 1000000
      batch_config:
        encoding:
          format: jsonl
          compression: gzip
        storage:
          root: file://
          prefix: ''
  - name: tap-carbon-intensity
    variant: meltano
    pip_url: git+https://gitlab.com/meltano/tap-carbon-intensity.git
  loaders:
  - name: target-bigquery
    namespace: target_bigquery
    pip_url: -e .
    capabilities:
    - about
    - batch
    - stream-maps
    - schema-flattening
    settings:
    - name: credentials_path
      kind: string
    - name: credentials_json
      kind: string
    - name: project
      kind: string
    - name: dataset
      kind: string
    - name: location
      kind: string
      value: US
    - name: batch_size
      kind: integer
      value: 500
    - name: fail_fast
      kind: boolean
      value: true
    - name: timeout
      kind: integer
      value: 600
    - name: denormalized
      kind: boolean
      value: false
    - name: method
      kind: options
      options:
      - label: Storage Write API
        value: storage_write_api
      - label: Batch Job
        value: batch_job
      - label: GCS Stage
        value: gcs_stage
      - label: Streaming Insert
        value: streaming_insert
      value: storage_write_api
    - name: generate_view
      kind: boolean
      value: false
    - name: bucket
      kind: string
    - name: partition_granularity
      kind: options
      options:
      - label: Year
        value: year
      - label: Month
        value: month
      - label: Day
        value: day
      - label: Hour
        value: hour
      value: month
    - name: partition_expiration_days
      kind: integer
    - name: cluster_on_key_properties
      kind: boolean
      value: false
    - name: column_name_transforms.lower
      kind: boolean
      value: false
    - name: column_name_transforms.quote
      kind: boolean
      value: false
    - name: column_name_transforms.add_underscore_when_invalid
      kind: boolean
      value: false
    - name: column_name_transforms.snake_case
      kind: boolean
      value: false
    - name: column_name_transforms.replace_period_with_underscore
      kind: boolean
      value: false
    - name: options.storage_write_batch_mode
      kind: boolean
      value: false
    - name: options.process_pool
      kind: boolean
      value: false
    - name: options.max_workers
      kind: integer
      value: false
    - name: upsert
      value: false
    - name: overwrite
      value: false
    - name: dedupe_before_upsert
      value: false
    - name: schema_resolver_version
      kind: integer
      value: 1
    config:
      project: ${BQ_PROJECT}
      method: storage_write_api
      denormalized: true
      credentials_path: ${HOME}/.gcp/dwh-credentials.json
      dataset: raw_testing
      batch_size: 500
      timeout: 300