Skip to content

Commit 64fa1d7

Browse files
yruslanyruslan
committed
#757 Add gpg_private_key and gpg_private_key_passphrase options to enable transparent decryption of GPG-encrypted files.
1 parent e035225 commit 64fa1d7

18 files changed

Lines changed: 235 additions & 56 deletions

File tree

‎README.md‎

Lines changed: 23 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1678,17 +1678,33 @@ You can chain multiple transformers by providing transformer classes as a comma-
16781678
3. Custom transformers run before binary properties are calculated. So inserting or removing fields changes the copybook
16791679
layout.
16801680
1681+
### GPG Transparent Decryption (Experimental)
1682+
1683+
Cobrix supports transparent decryption of GPG/PGP-encrypted files. You just need to provide the content of ASCII-armored
1684+
private key, and the passphrase (if non-empty).
1685+
1686+
```scala
1687+
val df = spark.read
1688+
.format("cobol")
1689+
.option("copybook", someCopybook)
1690+
.option("gpg_private_key", gpgPrivateKey)
1691+
.option("gpg_private_key_passphrase", gpgPrivateKeyPassphrase)
1692+
.load("/some/path")
1693+
```
1694+
16811695
## Summary of all available options
16821696
16831697
##### File reading options
16841698
1685-
| Option (usage example) | Description |
1686-
|----------------------------------------|:---------------------------------------------------------------------------------------------------------------|
1687-
| .option("data_paths", "/path1,/path2") | Allows loading data from multiple unrelated paths on the same filesystem. |
1688-
| .option("file_start_offset", "0") | Specifies the number of bytes to skip at the beginning of each file. |
1689-
| .option("file_end_offset", "0") | Specifies the number of bytes to skip at the end of each file. |
1690-
| .option("record_start_offset", "0") | Specifies the number of bytes to skip at the beginning of each record before applying copybook fields to data. |
1691-
| .option("record_end_offset", "0") | Specifies the number of bytes to skip at the end of each record after applying copybook fields to data. |
1699+
| Option (usage example) | Description |
1700+
|-----------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------|
1701+
| .option("data_paths", "/path1,/path2") | Allows loading data from multiple unrelated paths on the same filesystem. |
1702+
| .option("file_start_offset", "0") | Specifies the number of bytes to skip at the beginning of each file. |
1703+
| .option("file_end_offset", "0") | Specifies the number of bytes to skip at the end of each file. |
1704+
| .option("record_start_offset", "0") | Specifies the number of bytes to skip at the beginning of each record before applying copybook fields to data. |
1705+
| .option("record_end_offset", "0") | Specifies the number of bytes to skip at the end of each record after applying copybook fields to data. |
1706+
| .option("gpg_private_key", ascGpgKeyContent) | Specifies the ASCII-armored GPG private key for decrypting data files. |
1707+
| .option("gpg_private_key_passphrase", "passphrase") | Specifies the passphrase for the ASCII-armored GPG private key used for decrypting data files. If not specified, empty passphrase will be used. |
16921708
16931709
##### Copybook parsing options
16941710

‎cobol-parser/src/main/scala/za/co/absa/cobrix/cobol/reader/parameters/CobolParameters.scala‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,8 @@ import za.co.absa.cobrix.cobol.reader.policies.SchemaRetentionPolicy.SchemaReten
3838
* @param asciiCharset A charset for ASCII data
3939
* @param fieldCodePage Specifies a mapping between a field name and the code page
4040
* @param isUtf16BigEndian If true UTF-16 is considered big-endian.
41+
* @param gpgPrivateKey GPG private key content
42+
* @param gpgPrivateKeyPassphrase GPG private key passphrase
4143
* @param floatingPointFormat A format of floating-point numbers
4244
* @param recordStartOffset A number of bytes to skip at the beginning of the record before parsing a record according to a copybook
4345
* @param recordEndOffset A number of bytes to skip at the end of each record
@@ -84,6 +86,8 @@ case class CobolParameters(
8486
asciiCharset: Option[String],
8587
fieldCodePage: Map[String, String],
8688
isUtf16BigEndian: Boolean,
89+
gpgPrivateKey: Option[String],
90+
gpgPrivateKeyPassphrase: Option[String],
8791
floatingPointFormat: FloatingPointFormat,
8892
recordStartOffset: Int,
8993
recordEndOffset: Int,

‎cobol-parser/src/main/scala/za/co/absa/cobrix/cobol/reader/parameters/CobolParametersParser.scala‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,8 @@ object CobolParametersParser extends Logging {
6868
val PARAM_RECORD_TRAILER_NAME2 = "file_trailer_field"
6969
val PARAM_IS_XCOM = "is_xcom"
7070
val PARAM_IS_TEXT = "is_text"
71+
val PARAM_GPG_PRIVATE_KEY = "gpg_private_key"
72+
val PARAM_GPG_PRIVATE_KEY_PASSPHRASE = "gpg_private_key_passphrase"
7173

7274
// Schema transformation parameters
7375
val PARAM_GENERATE_RECORD_ID = "generate_record_id"
@@ -302,6 +304,9 @@ object CobolParametersParser extends Logging {
302304
)
303305
}
304306

307+
val gpgPrivateKey = params.get(PARAM_GPG_PRIVATE_KEY)
308+
val gpgPrivateKeyPassphraseOpt = params.get(PARAM_GPG_PRIVATE_KEY_PASSPHRASE)
309+
305310
val variableSizeOccursPolicy = VariableSizeOccursPolicy(params.getOrElse(PARAM_VARIABLE_SIZE_OCCURS, "false"))
306311

307312
val writerParameters = if (isWriter) {
@@ -326,6 +331,8 @@ object CobolParametersParser extends Logging {
326331
asciiCharset,
327332
getFieldCodepageMap(params),
328333
params.getOrElse(PARAM_IS_UTF16_BIG_ENDIAN, "true").toBoolean,
334+
gpgPrivateKey,
335+
gpgPrivateKeyPassphraseOpt,
329336
getFloatingPointFormat(params),
330337
params.getOrElse(PARAM_RECORD_START_OFFSET, "0").toInt,
331338
params.getOrElse(PARAM_RECORD_END_OFFSET, "0").toInt,
@@ -493,6 +500,8 @@ object CobolParametersParser extends Logging {
493500
asciiCharset = parameters.asciiCharset,
494501
fieldCodePage = parameters.fieldCodePage,
495502
isUtf16BigEndian = parameters.isUtf16BigEndian,
503+
gpgPrivateKey = parameters.gpgPrivateKey,
504+
gpgPrivateKeyPassphrase = parameters.gpgPrivateKeyPassphrase,
496505
floatingPointFormat = parameters.floatingPointFormat,
497506
redefineRuleExpressions = ruleExpressionMap,
498507
variableSizeOccurs = parameters.variableSizeOccurs,

‎cobol-parser/src/main/scala/za/co/absa/cobrix/cobol/reader/parameters/ReaderParameters.scala‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,8 @@ import za.co.absa.cobrix.cobol.reader.policies.SchemaRetentionPolicy.SchemaReten
3737
* @param asciiCharset A charset for ASCII data
3838
* @param fieldCodePage Specifies a mapping between a field name and the code page
3939
* @param isUtf16BigEndian If true UTF-16 strings are considered big-endian.
40+
* @param gpgPrivateKey GPG private key content
41+
* @param gpgPrivateKeyPassphrase GPG private key passphrase
4042
* @param floatingPointFormat A format of floating-point numbers
4143
* @param redefineRuleExpressions A map of REDEFINE field names to expressions that determine which redefine alternative to use when parsing records.
4244
* @param variableSizeOccurs Specifies how to handle OCCURS DEPENDING ON when the actual number of elements in arrays is less than the maximum array size
@@ -95,6 +97,8 @@ case class ReaderParameters(
9597
asciiCharset: Option[String] = None,
9698
fieldCodePage: Map[String, String] = Map.empty[String, String],
9799
isUtf16BigEndian: Boolean = true,
100+
gpgPrivateKey: Option[String] = None,
101+
gpgPrivateKeyPassphrase: Option[String] = None,
98102
floatingPointFormat: FloatingPointFormat = FloatingPointFormat.IBM,
99103
redefineRuleExpressions: Map[String, ExpressionEvaluator] = Map.empty,
100104
variableSizeOccurs: VariableSizeOccursPolicy = VariableSizeOccursPolicy.MaxSize,

‎data/test41_copybook.cob‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
****************************************************************************
2+
* *
3+
* Copyright 2018 ABSA Group Limited *
4+
* *
5+
* Licensed under the Apache License, Version 2.0 (the "License"); *
6+
* you may not use this file except in compliance with the License. *
7+
* You may obtain a copy of the License at *
8+
* *
9+
* http://www.apache.org/licenses/LICENSE-2.0 *
10+
* *
11+
* Unless required by applicable law or agreed to in writing, software *
12+
* distributed under the License is distributed on an "AS IS" BASIS, *
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. *
14+
* See the License for the specific language governing permissions and *
15+
* limitations under the License. *
16+
* *
17+
****************************************************************************
18+
19+
01 RECORD.
20+
05 ID PIC S9(4) COMP.
21+
05 COMPANY.
22+
10 SHORT-NAME PIC X(10).
23+
10 COMPANY-ID-NUM PIC 9(5) COMP-3.
24+
10 COMPANY-ID-STR
25+
REDEFINES COMPANY-ID-NUM PIC X(3).
26+
05 METADATA.
27+
10 CLIENTID PIC X(15).
28+
10 REGISTRATION-NUM PIC X(10).
29+
10 NUMBER-OF-ACCTS PIC 9(03) COMP-3.
30+
10 ACCOUNT.
31+
12 ACCOUNT-DETAIL OCCURS 80
32+
DEPENDING ON NUMBER-OF-ACCTS.
33+
15 ACCOUNT-NUMBER PIC X(24).
34+
15 ACCOUNT-TYPE-N PIC 9(5) COMP-3.
35+
15 ACCOUNT-TYPE-X REDEFINES
36+
ACCOUNT-TYPE-N PIC X(3).
37+


‎data/test41_data/example.bin.gpg‎

769 Bytes
Binary file not shown.

‎data/test41_expected/test41a.txt‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
{"ID":1,"COMPANY":{"SHORT_NAME":"FOO INCORP","COMPANY_ID_NUM":0,"COMPANY_ID_STR":""},"METADATA":{"CLIENTID":"","REGISTRATION_NUM":"","NUMBER_OF_ACCTS":1,"ACCOUNT":{"ACCOUNT_DETAIL":[{"ACCOUNT_NUMBER":"000000000000001100220033","ACCOUNT_TYPE_N":0,"ACCOUNT_TYPE_X":""}]}}}
2+
{"ID":2,"COMPANY":{"SHORT_NAME":"BARCOMPANY","COMPANY_ID_NUM":0,"COMPANY_ID_STR":""},"METADATA":{"CLIENTID":"","REGISTRATION_NUM":"","NUMBER_OF_ACCTS":1,"ACCOUNT":{"ACCOUNT_DETAIL":[{"ACCOUNT_NUMBER":"002000000022004000010001","ACCOUNT_TYPE_N":0,"ACCOUNT_TYPE_X":""}]}}}
3+
{"ID":3,"COMPANY":{"SHORT_NAME":"EXAMPLE.CO","COMPANY_ID_NUM":0,"COMPANY_ID_STR":""},"METADATA":{"CLIENTID":"","REGISTRATION_NUM":"","NUMBER_OF_ACCTS":1,"ACCOUNT":{"ACCOUNT_DETAIL":[{"ACCOUNT_NUMBER":"000000000000001234567890","ACCOUNT_TYPE_N":0,"ACCOUNT_TYPE_X":""}]}}}
4+
{"ID":4,"COMPANY":{"SHORT_NAME":"EXAMPLE330","COMPANY_ID_NUM":0,"COMPANY_ID_STR":""},"METADATA":{"CLIENTID":"","REGISTRATION_NUM":"","NUMBER_OF_ACCTS":2,"ACCOUNT":{"ACCOUNT_DETAIL":[{"ACCOUNT_NUMBER":"000000000000009876543210","ACCOUNT_TYPE_N":0,"ACCOUNT_TYPE_X":""},{"ACCOUNT_NUMBER":"000000000000001234555561","ACCOUNT_TYPE_N":1,"ACCOUNT_TYPE_X":""}]}}}
5+
{"ID":5,"COMPANY":{"SHORT_NAME":"EXAMPLE3","COMPANY_ID_NUM":0,"COMPANY_ID_STR":""},"METADATA":{"CLIENTID":"","REGISTRATION_NUM":"","NUMBER_OF_ACCTS":1,"ACCOUNT":{"ACCOUNT_DETAIL":[{"ACCOUNT_NUMBER":"000000012131415161718192","ACCOUNT_TYPE_N":0,"ACCOUNT_TYPE_X":""}]}}}
6+
{"ID":6,"COMPANY":{"SHORT_NAME":"EXAMPLE4","COMPANY_ID_NUM":0,"COMPANY_ID_STR":""},"METADATA":{"CLIENTID":"","REGISTRATION_NUM":"","NUMBER_OF_ACCTS":3,"ACCOUNT":{"ACCOUNT_DETAIL":[{"ACCOUNT_NUMBER":"000000000000002000400012","ACCOUNT_TYPE_N":0,"ACCOUNT_TYPE_X":""},{"ACCOUNT_NUMBER":"000000000000003000400102","ACCOUNT_TYPE_N":1,"ACCOUNT_TYPE_X":""},{"ACCOUNT_NUMBER":"000000005006001200301000","ACCOUNT_TYPE_N":2,"ACCOUNT_TYPE_X":""}]}}}
7+
{"ID":7,"COMPANY":{"SHORT_NAME":"EXAMPLE7","COMPANY_ID_NUM":0,"COMPANY_ID_STR":""},"METADATA":{"CLIENTID":"","REGISTRATION_NUM":"","NUMBER_OF_ACCTS":2,"ACCOUNT":{"ACCOUNT_DETAIL":[{"ACCOUNT_NUMBER":"000000100423412301203120","ACCOUNT_TYPE_N":0,"ACCOUNT_TYPE_X":""},{"ACCOUNT_NUMBER":"000000000030928973981723","ACCOUNT_TYPE_N":1,"ACCOUNT_TYPE_X":""}]}}}
8+
{"ID":8,"COMPANY":{"SHORT_NAME":"FOOBAR8","COMPANY_ID_NUM":0,"COMPANY_ID_STR":""},"METADATA":{"CLIENTID":"","REGISTRATION_NUM":"","NUMBER_OF_ACCTS":3,"ACCOUNT":{"ACCOUNT_DETAIL":[{"ACCOUNT_NUMBER":"000000389871238792010200","ACCOUNT_TYPE_N":0,"ACCOUNT_TYPE_X":""},{"ACCOUNT_NUMBER":"000000036719283719283713","ACCOUNT_TYPE_N":1,"ACCOUNT_TYPE_X":""},{"ACCOUNT_NUMBER":"000001992837819827389172","ACCOUNT_TYPE_N":2,"ACCOUNT_TYPE_X":""}]}}}
9+
{"ID":9,"COMPANY":{"SHORT_NAME":"DUMMY_CO9","COMPANY_ID_NUM":0,"COMPANY_ID_STR":""},"METADATA":{"CLIENTID":"","REGISTRATION_NUM":"","NUMBER_OF_ACCTS":1,"ACCOUNT":{"ACCOUNT_DETAIL":[{"ACCOUNT_NUMBER":"000000731928300100002312","ACCOUNT_TYPE_N":0,"ACCOUNT_TYPE_X":""}]}}}
10+
{"ID":10,"COMPANY":{"SHORT_NAME":"NEWEXCOM10","COMPANY_ID_NUM":0,"COMPANY_ID_STR":""},"METADATA":{"CLIENTID":"","REGISTRATION_NUM":"","NUMBER_OF_ACCTS":2,"ACCOUNT":{"ACCOUNT_DETAIL":[{"ACCOUNT_NUMBER":"000000004909239000000233","ACCOUNT_TYPE_N":2,"ACCOUNT_TYPE_X":""},{"ACCOUNT_NUMBER":"000000000984120003123900","ACCOUNT_TYPE_N":1,"ACCOUNT_TYPE_X":""}]}}}

‎data/test41_expected/test41b.txt‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
{"File_Id":0,"Record_Id":0,"Record_Byte_Length":2202,"ID":1,"COMPANY":{"SHORT_NAME":"FOO INCORP","COMPANY_ID_NUM":0,"COMPANY_ID_STR":""},"METADATA":{"CLIENTID":"","REGISTRATION_NUM":"","NUMBER_OF_ACCTS":1,"ACCOUNT":{"ACCOUNT_DETAIL":[{"ACCOUNT_NUMBER":"000000000000001100220033","ACCOUNT_TYPE_N":0,"ACCOUNT_TYPE_X":""}]}}}
2+
{"File_Id":0,"Record_Id":1,"Record_Byte_Length":2202,"ID":2,"COMPANY":{"SHORT_NAME":"BARCOMPANY","COMPANY_ID_NUM":0,"COMPANY_ID_STR":""},"METADATA":{"CLIENTID":"","REGISTRATION_NUM":"","NUMBER_OF_ACCTS":1,"ACCOUNT":{"ACCOUNT_DETAIL":[{"ACCOUNT_NUMBER":"002000000022004000010001","ACCOUNT_TYPE_N":0,"ACCOUNT_TYPE_X":""}]}}}
3+
{"File_Id":0,"Record_Id":2,"Record_Byte_Length":2202,"ID":3,"COMPANY":{"SHORT_NAME":"EXAMPLE.CO","COMPANY_ID_NUM":0,"COMPANY_ID_STR":""},"METADATA":{"CLIENTID":"","REGISTRATION_NUM":"","NUMBER_OF_ACCTS":1,"ACCOUNT":{"ACCOUNT_DETAIL":[{"ACCOUNT_NUMBER":"000000000000001234567890","ACCOUNT_TYPE_N":0,"ACCOUNT_TYPE_X":""}]}}}
4+
{"File_Id":0,"Record_Id":3,"Record_Byte_Length":2202,"ID":4,"COMPANY":{"SHORT_NAME":"EXAMPLE330","COMPANY_ID_NUM":0,"COMPANY_ID_STR":""},"METADATA":{"CLIENTID":"","REGISTRATION_NUM":"","NUMBER_OF_ACCTS":2,"ACCOUNT":{"ACCOUNT_DETAIL":[{"ACCOUNT_NUMBER":"000000000000009876543210","ACCOUNT_TYPE_N":0,"ACCOUNT_TYPE_X":""},{"ACCOUNT_NUMBER":"000000000000001234555561","ACCOUNT_TYPE_N":1,"ACCOUNT_TYPE_X":""}]}}}
5+
{"File_Id":0,"Record_Id":4,"Record_Byte_Length":2202,"ID":5,"COMPANY":{"SHORT_NAME":"EXAMPLE3","COMPANY_ID_NUM":0,"COMPANY_ID_STR":""},"METADATA":{"CLIENTID":"","REGISTRATION_NUM":"","NUMBER_OF_ACCTS":1,"ACCOUNT":{"ACCOUNT_DETAIL":[{"ACCOUNT_NUMBER":"000000012131415161718192","ACCOUNT_TYPE_N":0,"ACCOUNT_TYPE_X":""}]}}}
6+
{"File_Id":0,"Record_Id":5,"Record_Byte_Length":2202,"ID":6,"COMPANY":{"SHORT_NAME":"EXAMPLE4","COMPANY_ID_NUM":0,"COMPANY_ID_STR":""},"METADATA":{"CLIENTID":"","REGISTRATION_NUM":"","NUMBER_OF_ACCTS":3,"ACCOUNT":{"ACCOUNT_DETAIL":[{"ACCOUNT_NUMBER":"000000000000002000400012","ACCOUNT_TYPE_N":0,"ACCOUNT_TYPE_X":""},{"ACCOUNT_NUMBER":"000000000000003000400102","ACCOUNT_TYPE_N":1,"ACCOUNT_TYPE_X":""},{"ACCOUNT_NUMBER":"000000005006001200301000","ACCOUNT_TYPE_N":2,"ACCOUNT_TYPE_X":""}]}}}
7+
{"File_Id":0,"Record_Id":6,"Record_Byte_Length":2202,"ID":7,"COMPANY":{"SHORT_NAME":"EXAMPLE7","COMPANY_ID_NUM":0,"COMPANY_ID_STR":""},"METADATA":{"CLIENTID":"","REGISTRATION_NUM":"","NUMBER_OF_ACCTS":2,"ACCOUNT":{"ACCOUNT_DETAIL":[{"ACCOUNT_NUMBER":"000000100423412301203120","ACCOUNT_TYPE_N":0,"ACCOUNT_TYPE_X":""},{"ACCOUNT_NUMBER":"000000000030928973981723","ACCOUNT_TYPE_N":1,"ACCOUNT_TYPE_X":""}]}}}
8+
{"File_Id":0,"Record_Id":7,"Record_Byte_Length":2202,"ID":8,"COMPANY":{"SHORT_NAME":"FOOBAR8","COMPANY_ID_NUM":0,"COMPANY_ID_STR":""},"METADATA":{"CLIENTID":"","REGISTRATION_NUM":"","NUMBER_OF_ACCTS":3,"ACCOUNT":{"ACCOUNT_DETAIL":[{"ACCOUNT_NUMBER":"000000389871238792010200","ACCOUNT_TYPE_N":0,"ACCOUNT_TYPE_X":""},{"ACCOUNT_NUMBER":"000000036719283719283713","ACCOUNT_TYPE_N":1,"ACCOUNT_TYPE_X":""},{"ACCOUNT_NUMBER":"000001992837819827389172","ACCOUNT_TYPE_N":2,"ACCOUNT_TYPE_X":""}]}}}
9+
{"File_Id":0,"Record_Id":8,"Record_Byte_Length":2202,"ID":9,"COMPANY":{"SHORT_NAME":"DUMMY_CO9","COMPANY_ID_NUM":0,"COMPANY_ID_STR":""},"METADATA":{"CLIENTID":"","REGISTRATION_NUM":"","NUMBER_OF_ACCTS":1,"ACCOUNT":{"ACCOUNT_DETAIL":[{"ACCOUNT_NUMBER":"000000731928300100002312","ACCOUNT_TYPE_N":0,"ACCOUNT_TYPE_X":""}]}}}
10+
{"File_Id":0,"Record_Id":9,"Record_Byte_Length":2202,"ID":10,"COMPANY":{"SHORT_NAME":"NEWEXCOM10","COMPANY_ID_NUM":0,"COMPANY_ID_STR":""},"METADATA":{"CLIENTID":"","REGISTRATION_NUM":"","NUMBER_OF_ACCTS":2,"ACCOUNT":{"ACCOUNT_DETAIL":[{"ACCOUNT_NUMBER":"000000004909239000000233","ACCOUNT_TYPE_N":2,"ACCOUNT_TYPE_X":""},{"ACCOUNT_NUMBER":"000000000984120003123900","ACCOUNT_TYPE_N":1,"ACCOUNT_TYPE_X":""}]}}}

0 commit comments

Comments
 (0)