Updating with new dataset for 2021-11-09

Author: ErikTCrosby

R/review_public.R

@@ -26,7 +26,7 @@ quasi_identifiers = c("race_ethnicity_combined","sex","age_group")
 confidential_attributes = c("pos_spec_dt")
 
 #if I use a CSV then there's logic to change down below
-file_name <- "COVID_Cases_Public_Limited_20211012.parquet"
+file_name <- "COVID_Cases_Public_Limited_20211109.parquet"
 #file_name <- "github_test.csv"
 
 suppressed_file_name = paste(out_dir,"/",file_name,".suppressed.csv",sep="")

R/review_public_geo.R

@@ -50,7 +50,7 @@ linked_attributes = list(
   )
 
 #if I use a CSV then there's logic to change down below
-file_name <- "public_county_geography_2021-08-03.parquet"
+file_name <- "public_county_geography_2021-11-09.parquet"
 suppressed_file_name = paste(out_dir,"/",file_name,".suppressed.csv",sep="")
 detailed_file_name = paste(data_dir,"/",file_name,sep="")
 print(detailed_file_name)
@@ -61,6 +61,10 @@ df = read_parquet(detailed_file_name, as_data_frame = TRUE)
 #for some reason the dataframe from arrow makes sdc take forever and error, but if I make a new dataframe it works, todo figure it out
 data <- data.frame(df)
 
+save(df, file = "Geo_df.Rdata")
+remove(df)
+
+
 #summarize dataset
 result <- quick_summary(data, label="all_fields", qis=quasi_identifiers)
 
@@ -125,18 +129,24 @@ cat("\n\nWriting out a privacy eval report to:", paste(report_dir,"/",file_name,
 report(sdcObj, outdir = report_dir, filename = file_name,
        title = "SRRG Privacy Evaluation Report for Case Surveillance Public Data Set with Geography", internal = TRUE, verbose = FALSE)
 
+save(sdcObj, file = "Geo_sdcObj.Rdata")
+remove(sdcObj)
+
+save(data, file = "Geo_data.Rdata")
+remove(data)
+
 cat('Processing check for low population counties (rule #3), should be 0.\n')
 
 #weird way to read in a file to support utf characters in dataset #TODO fix
 fileIn=file(COUNTY_POP_FILE_NAME,open="rb",encoding="UTF-8-BOM")
 lines = readLines(fileIn)
-county_data = read.csv(text=lines, na.strings=c('NA',''), colClasses=c("state_county_combined_fips"="character"))
+county_data = read.csv(text=lines, na.strings=c('NA',''), colClasses=c("ï..state_county_combined_fips"="character"))
 
 #county_data = read.csv(COUNTY_POP_FILE_NAME, fileEncoding="UTF-8-BOM", na.strings=c('NA',''),colClasses=c("state_county_combined_fips"="character"))
+names(county_data)[names(county_data) == 'ï..state_county_combined_fips'] <- 'county_fips_code'
 names(county_data)[names(county_data) == 'state_county_combined_fips'] <- 'county_fips_code'
 names(county_data) <- tolower(names(county_data))
 county_data['state_abbr'] = state.abb[match(county_data$stname,state.name)]
-
 data_with_census = merge(x=data_na, y=county_data, by = 'county_fips_code', all.x = TRUE)
 
 #if I screw up the merge, that's bad
@@ -153,6 +163,9 @@ if (num_v > 0){
   print(violations[sample(num_v,min(num_v,5)),c(quasi_identifiers, 'sum_of_tot_pop')])
 }
 
+save(data_na, file = "Geo_data_na.Rdata")
+remove(data_na)
+
 #basically my logic here for rule #4 is that any county's population by sex, race, and/or ethnicity should be greater than or equal to 220. So I'll check each record that's not NA
 cat('\n\nProcessing check for low subpopulation demographics in counties (rule #4), should be 0, twice.\n')
 
@@ -202,6 +215,11 @@ if (num_v > 0){
   print(violations[sample(num_v,min(num_v,5)),])
 }
 
+save(data_with_census_case, file = "Geo_data_with_census_case.Rdata")
+remove(data_with_census_case)
+
+load(file = "Geo_data.Rdata")
+
 cat('\n\nProcessing check for county/state complementary offsets (rule #10), should be 0.\n')
 
 census_counties_per_state = aggregate(data=county_data, ctyname ~ state_abbr, function(x) length(unique(x)))

analysis/utility_summary_public.pdf

@@ -2708,3 +2708,131 @@ checking l-diversity manually for < pos_spec_dt >, found < 0 > l-diversity viola
 <0 rows> (or 0-length row.names)
 
 ```
+
+## 2020-10-25
+
+[DataExplorer Profile Report](../reports/COVID_Cases_Public_Limited_2021-10-25_parquet_profile.html)
+
+```
+[1] "C:/Users/rwe4/OneDrive - CDC/R_projects/covid_case_privacy_review/R"
+
+[1] "Quick summary:"
+                  all_fields  quasi_fields
+total_fields      12          3           
+total_records     35,638,780  35,638,780  
+total_cells       427,665,360 106,916,340 
+missing_fields    86,949,327  1,273,443   
+missing_pct       20%         1%          
+complete_fields   340,716,033 105,642,897 
+complete_pct      80%         99%         
+unknown_fields    27,947,596  11,608,609  
+unknown_pct       7%          11%         
+suppressed_fields 103         103         
+suppressed_pct    0%          0%          
+available_fields  312,768,334 94,034,185  
+available_pct     73%         88%    
+
+Utility summary:
+Total records in dataset: 35,638,780 
+                        suppressed suppressed_percent   missing missing_percent
+race_ethnicity_combined         12               0.0%   832,461            2.3%
+sex                             18               0.0%    74,860            0.2%
+age_group                       73               0.0%   366,122            1.0%
+records_with_any_field          73               0.0% 1,182,996            3.3%
+
+SDC summary for k-anon-level( 5 ).
+The input dataset consists of 35638780 rows and 12 variables.
+  --> Categorical key variables: race_ethnicity_combined, sex, age_group
+----------------------------------------------------------------------
+
+
+Information on categorical key variables:
+
+Reported is the number, mean size and size of the smallest category >0 for recoded variables.
+In parenthesis, the same statistics are shown for the unmodified data.
+Note: NA (missings) are counted as seperate categories!
+
+
+            Key Variable Number of categories         Mean size                Size of smallest (>0)          
+ race_ethnicity_combined                    8  (8)  3357568.286  (3357568.286)                 68000   (68000)
+                     sex                    4  (4) 11752874.000 (11752874.000)                   705     (705)
+               age_group                   10 (10)  3919176.111  (3919176.111)               1222842 (1222842)
+----------------------------------------------------------------------
+
+
+Risk measures:
+
+Number of observations with higher risk than the main part of the data: 104
+Expected number of re-identifications: 161.53 (0.00 %)
+
+k-anon violations ( 0 ) for k=( 5 ) and quasi-identifiers ( race_ethnicity_combined sex age_group ). If greater than zero violations, then here's 5 violations.
+
+checking l-diversity manually for < pos_spec_dt >, found < 0 > l-diversity violations.
+[1] race_ethnicity_combined sex                     age_group               pos_spec_dt            
+[5] fk                      ldiv                   
+<0 rows> (or 0-length row.names)
+```
+
+## 2021-11-09
+
+[DataExplorer Profile Report](../reports/COVID_Cases_Public_Limited_2021-11-09_parquet_profile.html)
+
+```
+[1] "C:/Users/rwe4/OneDrive - CDC/R_projects/covid_case_privacy_review/R"
+
+[1] "Quick summary:"
+                  all_fields  quasi_fields
+total_fields      12          3           
+total_records     37,532,072  37,532,072  
+total_cells       450,384,864 112,596,216 
+missing_fields    96,399,423  1,323,719   
+missing_pct       21%         1%          
+complete_fields   353,985,441 111,272,497 
+complete_pct      79%         99%         
+unknown_fields    29,121,517  12,249,731  
+unknown_pct       6%          11%         
+suppressed_fields 108         108         
+suppressed_pct    0%          0%          
+available_fields  324,863,816 99,022,658  
+available_pct     72%         88%         
+
+Utility summary:
+Total records in dataset: 37,532,072 
+                        suppressed suppressed_percent   missing missing_percent
+race_ethnicity_combined         13               0.0%   875,896            2.3%
+sex                             19               0.0%    76,966            0.2%
+age_group                       76               0.0%   370,857            1.0%
+records_with_any_field          76               0.0% 1,232,005            3.3%
+
+SDC summary for k-anon-level( 5 ).
+The input dataset consists of 37532072 rows and 12 variables.
+  --> Categorical key variables: race_ethnicity_combined, sex, age_group
+----------------------------------------------------------------------
+
+
+Information on categorical key variables:
+
+Reported is the number, mean size and size of the smallest category >0 for recoded variables.
+In parenthesis, the same statistics are shown for the unmodified data.
+Note: NA (missings) are counted as seperate categories!
+
+
+            Key Variable Number of categories         Mean size                Size of smallest (>0)          
+ race_ethnicity_combined                    8  (8)  3532905.429  (3532905.429)                 70494   (70494)
+                     sex                    4  (4) 12377060.333 (12377060.333)                   754     (754)
+               age_group                   10 (10)  4129015.444  (4129015.444)               1287049 (1287049)
+----------------------------------------------------------------------
+
+
+Risk measures:
+
+Number of observations with higher risk than the main part of the data: 114
+Expected number of re-identifications: 161.66 (0.00 %)
+
+k-anon violations ( 0 ) for k=( 5 ) and quasi-identifiers ( race_ethnicity_combined sex age_group ). If greater than zero violations, then here's 5 violations.
+
+checking l-diversity manually for < pos_spec_dt >, found < 0 > l-diversity violations.
+[1] race_ethnicity_combined sex                     age_group               pos_spec_dt            
+[5] fk                      ldiv                   
+<0 rows> (or 0-length row.names)
+'''