diff --git a/td_load/sftp_v2/README.md b/td_load/sftp_v2/README.md new file mode 100644 index 00000000..607eb401 --- /dev/null +++ b/td_load/sftp_v2/README.md @@ -0,0 +1,50 @@ +# Workflow: td_load Example (SFTP_V2) + +This example workflow ingests data in daily basis, using [Treasure Data's Data Connector for SFTP V2](https://docs.treasuredata.com/articles/#!int/sftp_v2-server-import-integration/q/sftp_v2/qid/12261/qp/1) with [td_load](https://docs.digdag.io/operators/td_load.html) operator. + +The workflow also uses [Secrets](https://docs.treasuredata.com/articles/#!pd/setting-workflow-secrets-from-td-console/q/secret/qp/2) feature, so that you don't have to include your datasource credentials to your workflow files. + +# How to Run + +First, you can upload the workflow and trigger the session manually. + + # Upload + $ td wf push td_load_example + +Second, please set SFTP credentials by `td wf secrets` command. We recommend you to use text file for setting secret_key_file. For more details, please see [digdag documentation](https://docs.digdag.io/command_reference.html#secrets) + + # Set Secrets + $ td wf secrets --project td_load_example --set sftp.host + $ td wf secrets --project td_load_example --set sftp.port + $ td wf secrets --project td_load_example --set sftp.user + $ td wf secrets --project td_load_example --set sftp.secret_key_passphrase + $ td wf secrets --project td_load_example --set sftp.secret_key_file=@secret_key_file.txt + + # Set Secrets on your local for testing + $ td wf secrets --local --set sftp.host + $ td wf secrets --local --set sftp.port + $ td wf secrets --local --set sftp.user + $ td wf secrets --local --set sftp.secret_key_passphrase + $ td wf secrets --local --set sftp.secret_key_file=@secret_key_file.txt + +Now you can reference these credentials by `${secret:}` syntax within yml file for `td_load` operator. + +- [config/daily_load.yml](config/daily_load.yml) + +Now, you can trigger the session manually. + + # Run + $ td wf start td_load_example daily_load --session now + +# Required Keys + +| Keys | Description | +| -------- | ----------- | +| host | Host name. | +| port | Port number. | +| user | User name. | +| path_prefix | Prefix of output paths. | + +# Next Step + +If you have any questions, please contact support@treasure-data.com. diff --git a/td_load/sftp_v2/config/daily_load.yml b/td_load/sftp_v2/config/daily_load.yml new file mode 100644 index 00000000..a6139988 --- /dev/null +++ b/td_load/sftp_v2/config/daily_load.yml @@ -0,0 +1,29 @@ +in: + type: sftp_v2 + host: ${secret:sftp.host} + port: ${secret:sftp.port} + auth_method: key_pair + #auth_method: password + user: ${secret:sftp.user} + secret_key_file: {"content": "${secret:sftp.secret_key_file}"} + secret_key_passphrase: ${secret:sftp.secret_key_passphrase} + user_directory_is_root: ${sftp.user_directory_is_root} + timeout: ${sftp.timeout} + path_prefix: "/path/file_${moment(session_time).format("YYYYMMDD")}.csv" + parser: + charset: UTF-8 + newline: LF + type: csv + delimiter: "," + quote: "\"" + escape: "\"" + trim_if_not_quoted: false + skip_header_lines: 1 + allow_extra_columns: false + allow_optional_columns: false + columns: + - {name: time, type: long} + - {name: id, type: long} + - {name: name, type: string} + - {name: category, type: string} +out: {} diff --git a/td_load/sftp_v2/daily_load.dig b/td_load/sftp_v2/daily_load.dig new file mode 100644 index 00000000..63b5f7ea --- /dev/null +++ b/td_load/sftp_v2/daily_load.dig @@ -0,0 +1,29 @@ +timezone: UTC + +schedule: + daily>: 02:00:00 + +sla: + time: 08:00 + +notice: + mail>: {data: Treasure Workflow Notification} + subject: This workflow is taking long time to finish + to: [me@example.com] + +_export: + td: + dest_db: dest_db + dest_table: dest_table + sftp: + user_directory_is_root: false + timeout: 600 + ++prepare_table: + td_ddl>: + database: ${td.dest_db} + create_tables: ["${td.dest_table}"] + ++load_step: + td_load>: config/daily_load.yml + database: ${td.dest_db} + table: ${td.dest_table} diff --git a/td_load/sftp_v2/secret_key_file.txt b/td_load/sftp_v2/secret_key_file.txt new file mode 100644 index 00000000..685d7e8f --- /dev/null +++ b/td_load/sftp_v2/secret_key_file.txt @@ -0,0 +1,11 @@ +-----BEGIN RSA PRIVATE KEY----- +Proc-Type: 4,ENCRYPTED +DEK-Info: ABCDEFGHIJKLMNOPQRSTUVWXYZ + +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB +CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC +DDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDD +EEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEE +ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ +-----END RSA PRIVATE KEY-----