Skip to content

Commit 2559553

Browse files
yruslanyruslan
committed
#757 Add a test for reading GPG-encrypted variable-record-length files with indexes.
1 parent 1d1332c commit 2559553

2 files changed

Lines changed: 32 additions & 4 deletions

File tree

‎data/test41_expected/test41c.txt‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
{"File_Id":0,"Record_Id":0,"Record_Byte_Length":2202,"ID":1,"COMPANY":{"SHORT_NAME":"FOO INCORP","COMPANY_ID_NUM":0,"COMPANY_ID_STR":""},"METADATA":{"CLIENTID":"","REGISTRATION_NUM":"","NUMBER_OF_ACCTS":1,"ACCOUNT":{"ACCOUNT_DETAIL":[{"ACCOUNT_NUMBER":"000000000000001100220033","ACCOUNT_TYPE_N":0,"ACCOUNT_TYPE_X":""}]}}}
2+
{"File_Id":0,"Record_Id":1,"Record_Byte_Length":2202,"ID":2,"COMPANY":{"SHORT_NAME":"BARCOMPANY","COMPANY_ID_NUM":0,"COMPANY_ID_STR":""},"METADATA":{"CLIENTID":"","REGISTRATION_NUM":"","NUMBER_OF_ACCTS":1,"ACCOUNT":{"ACCOUNT_DETAIL":[{"ACCOUNT_NUMBER":"002000000022004000010001","ACCOUNT_TYPE_N":0,"ACCOUNT_TYPE_X":""}]}}}
3+
{"File_Id":0,"Record_Id":2,"Record_Byte_Length":2202,"ID":3,"COMPANY":{"SHORT_NAME":"EXAMPLE.CO","COMPANY_ID_NUM":0,"COMPANY_ID_STR":""},"METADATA":{"CLIENTID":"","REGISTRATION_NUM":"","NUMBER_OF_ACCTS":1,"ACCOUNT":{"ACCOUNT_DETAIL":[{"ACCOUNT_NUMBER":"000000000000001234567890","ACCOUNT_TYPE_N":0,"ACCOUNT_TYPE_X":""}]}}}
4+
{"File_Id":0,"Record_Id":3,"Record_Byte_Length":2202,"ID":4,"COMPANY":{"SHORT_NAME":"EXAMPLE330","COMPANY_ID_NUM":0,"COMPANY_ID_STR":""},"METADATA":{"CLIENTID":"","REGISTRATION_NUM":"","NUMBER_OF_ACCTS":2,"ACCOUNT":{"ACCOUNT_DETAIL":[{"ACCOUNT_NUMBER":"000000000000009876543210","ACCOUNT_TYPE_N":0,"ACCOUNT_TYPE_X":""},{"ACCOUNT_NUMBER":"000000000000001234555561","ACCOUNT_TYPE_N":1,"ACCOUNT_TYPE_X":""}]}}}
5+
{"File_Id":0,"Record_Id":4,"Record_Byte_Length":2202,"ID":5,"COMPANY":{"SHORT_NAME":"EXAMPLE3","COMPANY_ID_NUM":0,"COMPANY_ID_STR":""},"METADATA":{"CLIENTID":"","REGISTRATION_NUM":"","NUMBER_OF_ACCTS":1,"ACCOUNT":{"ACCOUNT_DETAIL":[{"ACCOUNT_NUMBER":"000000012131415161718192","ACCOUNT_TYPE_N":0,"ACCOUNT_TYPE_X":""}]}}}
6+
{"File_Id":0,"Record_Id":5,"Record_Byte_Length":2202,"ID":6,"COMPANY":{"SHORT_NAME":"EXAMPLE4","COMPANY_ID_NUM":0,"COMPANY_ID_STR":""},"METADATA":{"CLIENTID":"","REGISTRATION_NUM":"","NUMBER_OF_ACCTS":3,"ACCOUNT":{"ACCOUNT_DETAIL":[{"ACCOUNT_NUMBER":"000000000000002000400012","ACCOUNT_TYPE_N":0,"ACCOUNT_TYPE_X":""},{"ACCOUNT_NUMBER":"000000000000003000400102","ACCOUNT_TYPE_N":1,"ACCOUNT_TYPE_X":""},{"ACCOUNT_NUMBER":"000000005006001200301000","ACCOUNT_TYPE_N":2,"ACCOUNT_TYPE_X":""}]}}}
7+
{"File_Id":0,"Record_Id":6,"Record_Byte_Length":2202,"ID":7,"COMPANY":{"SHORT_NAME":"EXAMPLE7","COMPANY_ID_NUM":0,"COMPANY_ID_STR":""},"METADATA":{"CLIENTID":"","REGISTRATION_NUM":"","NUMBER_OF_ACCTS":2,"ACCOUNT":{"ACCOUNT_DETAIL":[{"ACCOUNT_NUMBER":"000000100423412301203120","ACCOUNT_TYPE_N":0,"ACCOUNT_TYPE_X":""},{"ACCOUNT_NUMBER":"000000000030928973981723","ACCOUNT_TYPE_N":1,"ACCOUNT_TYPE_X":""}]}}}
8+
{"File_Id":0,"Record_Id":7,"Record_Byte_Length":2202,"ID":8,"COMPANY":{"SHORT_NAME":"FOOBAR8","COMPANY_ID_NUM":0,"COMPANY_ID_STR":""},"METADATA":{"CLIENTID":"","REGISTRATION_NUM":"","NUMBER_OF_ACCTS":3,"ACCOUNT":{"ACCOUNT_DETAIL":[{"ACCOUNT_NUMBER":"000000389871238792010200","ACCOUNT_TYPE_N":0,"ACCOUNT_TYPE_X":""},{"ACCOUNT_NUMBER":"000000036719283719283713","ACCOUNT_TYPE_N":1,"ACCOUNT_TYPE_X":""},{"ACCOUNT_NUMBER":"000001992837819827389172","ACCOUNT_TYPE_N":2,"ACCOUNT_TYPE_X":""}]}}}
9+
{"File_Id":0,"Record_Id":8,"Record_Byte_Length":2202,"ID":9,"COMPANY":{"SHORT_NAME":"DUMMY_CO9","COMPANY_ID_NUM":0,"COMPANY_ID_STR":""},"METADATA":{"CLIENTID":"","REGISTRATION_NUM":"","NUMBER_OF_ACCTS":1,"ACCOUNT":{"ACCOUNT_DETAIL":[{"ACCOUNT_NUMBER":"000000731928300100002312","ACCOUNT_TYPE_N":0,"ACCOUNT_TYPE_X":""}]}}}
10+
{"File_Id":0,"Record_Id":9,"Record_Byte_Length":2202,"ID":10,"COMPANY":{"SHORT_NAME":"NEWEXCOM10","COMPANY_ID_NUM":0,"COMPANY_ID_STR":""},"METADATA":{"CLIENTID":"","REGISTRATION_NUM":"","NUMBER_OF_ACCTS":2,"ACCOUNT":{"ACCOUNT_DETAIL":[{"ACCOUNT_NUMBER":"000000004909239000000233","ACCOUNT_TYPE_N":2,"ACCOUNT_TYPE_X":""},{"ACCOUNT_NUMBER":"000000000984120003123900","ACCOUNT_TYPE_N":1,"ACCOUNT_TYPE_X":""}]}}}

spark-cobol/src/test/scala/za/co/absa/cobrix/spark/cobol/source/integration/Test45PgpEncryptedFilesSpec.scala renamed to spark-cobol/src/test/scala/za/co/absa/cobrix/spark/cobol/source/integration/Test45GpgEncryptedFilesSpec.scala

Lines changed: 22 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ import za.co.absa.cobrix.spark.cobol.utils.{FileUtils, ResourceUtils}
2525
import java.nio.charset.StandardCharsets
2626
import java.nio.file.{Files, Paths}
2727

28-
class Test45PgpEncryptedFilesSpec extends AnyWordSpec with SparkTestBase with BinaryFileFixture with SimpleComparisonBase {
28+
class Test45GpgEncryptedFilesSpec extends AnyWordSpec with SparkTestBase with BinaryFileFixture with SimpleComparisonBase {
2929
private implicit val logger: Logger = LoggerFactory.getLogger(this.getClass)
3030

3131
private val exampleName = "Test41"
@@ -35,6 +35,8 @@ class Test45PgpEncryptedFilesSpec extends AnyWordSpec with SparkTestBase with Bi
3535
private val actualResultsAPath = "../data/test41_expected/test41a_actual.txt"
3636
private val expectedResultsBPath = "../data/test41_expected/test41b.txt"
3737
private val actualResultsBPath = "../data/test41_expected/test41b_actual.txt"
38+
private val expectedResultsCPath = "../data/test41_expected/test41c.txt"
39+
private val actualResultsCPath = "../data/test41_expected/test41c_actual.txt"
3840

3941
"gpg encrypted files" should {
4042
"load normally a fixed-record-length file" in {
@@ -44,9 +46,6 @@ class Test45PgpEncryptedFilesSpec extends AnyWordSpec with SparkTestBase with Bi
4446
.option("copybook", inputCopybookPath)
4547
.option("gpg_private_key", gpgPrivateKey)
4648
.option("pedantic", "true")
47-
//.option("generate_record_id", "true")
48-
//.option("enable_indexes", "false")
49-
//.option("debug_ignore_file_size", "true")
5049
.load(inputDataPath)
5150

5251
val actual = df.toJSON.take(60)
@@ -78,5 +77,24 @@ class Test45PgpEncryptedFilesSpec extends AnyWordSpec with SparkTestBase with Bi
7877
}
7978
}
8079

80+
"load normally a variable-record-length file with indexes" in {
81+
val gpgPrivateKey = ResourceUtils.readResourceAsString("/test/test_gpg_key.asc")
82+
val df = spark.read
83+
.format("cobol")
84+
.option("copybook", inputCopybookPath)
85+
.option("gpg_private_key", gpgPrivateKey)
86+
.option("generate_record_id", "true")
87+
.option("input_split_records", 1)
88+
.option("pedantic", "true")
89+
.load(inputDataPath)
90+
91+
val actual = df.toJSON.take(60)
92+
val expected = Files.readAllLines(Paths.get(expectedResultsCPath), StandardCharsets.ISO_8859_1).toArray
93+
94+
if (!actual.sameElements(expected)) {
95+
FileUtils.writeStringsToFile(actual, actualResultsCPath)
96+
assert(false, s"The actual data doesn't match what is expected for $exampleName example. Please compare contents of $expectedResultsCPath to $actualResultsCPath for details.")
97+
}
98+
}
8199
}
82100
}

0 commit comments

Comments
 (0)