added directory and files function

Author: RohanVashisht1234

Database_SQL_Files/database_schema.sql

@@ -70,6 +70,7 @@ CREATE TABLE releases (
   -- Making this only 30 characters limit, it shouldn't be more than that.
   minimum_zig_version VARCHAR(30) NOT NULL,
   readme_url TEXT NOT NULL,
+  directory_files TEXT NOT NULL DEFAULT '',
   FOREIGN KEY (repo_id) REFERENCES repos (id) ON DELETE CASCADE,
   UNIQUE (repo_id, version)
 );

src/constants.rs

@@ -31,4 +31,5 @@ pub mod limits {
     pub const RELEASE_MIN_ZIG_VERSION_MAX_LEN: usize = 30;
     pub const RELEASE_DEPENDENCY_FIELD_MAX_LEN: usize = 260;
     pub const INDEX_SECTION_NAME_MAX_LEN: usize = 10;
+    pub const RELEASE_DIRECTORY_FILES_MAX_LEN: usize = 5000;
 }

src/github/cron_update_helper.rs

@@ -12,39 +12,48 @@ struct NeedsUpdateRow {
     type_of_repo: String,
 }
 
-async fn fetch_root_folder_directory_files(
+pub async fn fetch_root_folder_directory_files(
     client: &reqwest::Client,
     user_name: String,
     repo_name: String,
-) -> String {
-    // "https://api.github.com/repos/zigistry/zigistry/contents"
+    branch_or_tag: String,
+) -> Result<String, Box<dyn Error + Send + Sync>> {
+    let url = format!(
+        "https://api.github.com/repos/{user_name}/{repo_name}/contents?ref={branch_or_tag}"
+    );
     let response = client
-        .get(format!(
-            "https://api.github.com/repos/{user_name}/{repo_name}/contents"
-        ))
+        .get(&url)
         .header("User-Agent", "zigistry")
         .header("Authorization", &*GITHUB_KEY)
         .send()
-        .await
-        .unwrap();
-    let response_json: Vec<serde_json::Value> = response.json().await.unwrap();
+        .await?;
+    if !response.status().is_success() {
+        return Err(format!("GitHub API returned {}", response.status()).into());
+    }
+    let response_json: Vec<serde_json::Value> = response.json().await?;
     let mut directories = Vec::new();
     let mut files = Vec::new();
 
     for thing in response_json {
-        let name = thing["name"].as_str().unwrap();
-        let kind = thing["type"].as_str().unwrap();
+        let name = match thing["name"].as_str() {
+            Some(name) => name.to_string(),
+            None => continue,
+        };
+        let kind = match thing["type"].as_str() {
+            Some(kind) => kind,
+            None => continue,
+        };
 
         match kind {
-            "dir" => directories.push(name.to_string()),
-            "file" => files.push(name.to_string()),
+            "dir" => directories.push(name),
+            "file" => files.push(name),
             _ => {}
         }
     }
     let dirs_string = directories.join("\n");
     let files_string = files.join("\n");
     let join_both_strings = dirs_string + "\n\n" + &files_string;
-    join_both_strings
+    Ok(join_both_strings)
 }
 
 fn parse_github_repo_id(repo_id: &str) -> Option<(String, String)> {
@@ -196,3 +205,17 @@ pub async fn run_cron_update_once(pool: Arc<Connection>) -> Result<(), Box<dyn E
 
     Ok(())
 }
+
+#[tokio::test]
+async fn test_fetch_root_folder_directory_files() {
+    let client = reqwest::Client::new();
+    let result = fetch_root_folder_directory_files(
+        &client,
+        "zigistry".to_string(),
+        "zigistry".to_string(),
+        "main".to_string(),
+    )
+    .await
+    .unwrap();
+    println!("directory_files output:\n{result}");
+}

src/github/github_data.rs

@@ -9,6 +9,7 @@ pub struct ReleaseData {
     pub published_at: String,
     pub minimum_zig_version: String,
     pub readme_url: String,
+    pub directory_files: String,
     pub dependencies: Vec<Dependency>,
 }
 
@@ -22,5 +23,6 @@ pub struct RepoData {
     pub readme_content: String,
     pub build_zig_zon_version: String,
     pub build_zig_zon_dependencies: Vec<Dependency>,
+    pub default_branch_directory_files: String,
     pub releases: Vec<ReleaseData>,
 }

src/github/mod.rs

@@ -198,15 +198,21 @@ pub async fn get_repo_data(
         default_branch_name.as_ref()
     };
 
-    let (build_zig_zon_data, (readme_url, readme_content)) = tokio::join!(
+    let (build_zig_zon_data, (readme_url, readme_content), default_branch_directory_files) = tokio::join!(
         get_build_zig_zon_data_wrapper(&repository.owner.login, &repository.name, branch, client),
         get_readme_url_and_content(
             &repository.owner.login,
             &repository.name,
             branch,
             true,
             client
-        )
+        ),
+        fetch_root_folder_directory_files_wrapper(
+            &repository.owner.login,
+            &repository.name,
+            branch,
+            client
+        ),
     );
 
     let (readme_url, readme_content) = match (readme_url, readme_content) {
@@ -223,20 +229,24 @@ pub async fn get_repo_data(
         let release_clone = release.clone();
 
         async move {
-            let (readme_url, _) =
-                match get_readme_url_and_content(&owner, &name, &tag, false, client).await {
-                    (Some(url), _) => (url, String::new()),
-                    _ => ("404 unable to find readme.".to_string(), String::new()),
-                };
-
-            let bzz_results = get_build_zig_zon_data_wrapper(&owner, &name, &tag, client).await;
+            let ((readme_url, _), bzz_results, directory_files) = tokio::join!(
+                async {
+                    match get_readme_url_and_content(&owner, &name, &tag, false, client).await {
+                        (Some(url), _) => (url, String::new()),
+                        _ => ("404 unable to find readme.".to_string(), String::new()),
+                    }
+                },
+                get_build_zig_zon_data_wrapper(&owner, &name, &tag, client),
+                fetch_root_folder_directory_files_wrapper(&owner, &name, &tag, client),
+            );
 
             ReleaseData {
                 tag_name: release_clone.tag_name,
                 is_prerelease: release_clone.is_prerelease,
                 published_at: release_clone.published_at,
                 minimum_zig_version: bzz_results.0,
                 readme_url,
+                directory_files,
                 dependencies: bzz_results.1,
             }
         }
@@ -261,6 +271,7 @@ pub async fn get_repo_data(
         readme_content: readme_processed_content,
         build_zig_zon_version: build_zig_zon_data.0,
         build_zig_zon_dependencies: build_zig_zon_data.1,
+        default_branch_directory_files,
         releases,
     }
 }
@@ -275,6 +286,7 @@ pub async fn persist_repo_data(transaction: &Transaction, data: RepoData) {
         readme_content,
         build_zig_zon_version,
         build_zig_zon_dependencies,
+        default_branch_directory_files,
         releases,
     } = data;
 
@@ -439,13 +451,14 @@ pub async fn persist_repo_data(transaction: &Transaction, data: RepoData) {
         .query(
             r#"
             INSERT INTO releases
-                (repo_id, version, is_prerelease, published_at, minimum_zig_version, readme_url)
-            VALUES(?, ?, ?, ?, ?, ?)
+                (repo_id, version, is_prerelease, published_at, minimum_zig_version, readme_url, directory_files)
+            VALUES(?, ?, ?, ?, ?, ?, ?)
             ON CONFLICT(repo_id, version) DO UPDATE SET
                 is_prerelease = excluded.is_prerelease,
                 published_at = excluded.published_at,
                 minimum_zig_version = excluded.minimum_zig_version,
-                readme_url = excluded.readme_url
+                readme_url = excluded.readme_url,
+                directory_files = excluded.directory_files
             RETURNING id
             "#,
             params![
@@ -461,6 +474,10 @@ pub async fn persist_repo_data(transaction: &Transaction, data: RepoData) {
                     limits::RELEASE_MIN_ZIG_VERSION_MAX_LEN
                 ),
                 readme_url.clone(),
+                truncate_to_char_limit(
+                    &default_branch_directory_files,
+                    limits::RELEASE_DIRECTORY_FILES_MAX_LEN
+                ),
             ],
         )
         .await
@@ -519,13 +536,14 @@ pub async fn persist_repo_data(transaction: &Transaction, data: RepoData) {
             .query(
                 r#"
                 INSERT INTO releases
-                    (repo_id, version, is_prerelease, published_at, minimum_zig_version, readme_url)
-                VALUES(?, ?, ?, ?, ?, ?)
+                    (repo_id, version, is_prerelease, published_at, minimum_zig_version, readme_url, directory_files)
+                VALUES(?, ?, ?, ?, ?, ?, ?)
                 ON CONFLICT(repo_id, version) DO UPDATE SET
                     is_prerelease = excluded.is_prerelease,
                     published_at = excluded.published_at,
                     minimum_zig_version = excluded.minimum_zig_version,
-                    readme_url = excluded.readme_url
+                    readme_url = excluded.readme_url,
+                    directory_files = excluded.directory_files
                 RETURNING id
                 "#,
                 params![
@@ -538,6 +556,10 @@ pub async fn persist_repo_data(transaction: &Transaction, data: RepoData) {
                         limits::RELEASE_MIN_ZIG_VERSION_MAX_LEN
                     ),
                     release_data.readme_url,
+                    truncate_to_char_limit(
+                        &release_data.directory_files,
+                        limits::RELEASE_DIRECTORY_FILES_MAX_LEN
+                    ),
                 ],
             )
             .await
@@ -1074,6 +1096,25 @@ async fn get_build_zig_zon_data_wrapper(
     }
 }
 
+async fn fetch_root_folder_directory_files_wrapper(
+    owner_name: &str,
+    repo_name: &str,
+    branch_or_tag: &str,
+    client: &reqwest::Client,
+) -> String {
+    match cron_update_helper::fetch_root_folder_directory_files(
+        client,
+        owner_name.to_string(),
+        repo_name.to_string(),
+        branch_or_tag.to_string(),
+    )
+    .await
+    {
+        Ok(files) => files,
+        Err(_) => String::new(),
+    }
+}
+
 /// I have added this new client, which is much more optimized
 /// becuase, initially, I wasn't adding any timeouts.
 fn create_optimized_client() -> reqwest::Client {