diff --git a/generate-assets/src/bin/generate.rs b/generate-assets/src/bin/generate.rs index 8739a4e046..eeb4c170c9 100644 --- a/generate-assets/src/bin/generate.rs +++ b/generate-assets/src/bin/generate.rs @@ -14,8 +14,9 @@ fn main() -> anyhow::Result<()> { let asset_dir = std::env::args().nth(1).unwrap(); let content_dir = std::env::args().nth(2).unwrap(); + let no_update = std::env::args().nth(3) == Some(String::from("--no-update")); - let db = prepare_crates_db()?; + let db = prepare_crates_db(!no_update)?; let github_client = { // This should be configured in CI, but it's not mandatory if running locally @@ -61,6 +62,7 @@ fn main() -> anyhow::Result<()> { /// Sort the assets in the section so that: /// - Assets that have been manually assigned an order in `bevy-assets` are first /// - Assets that are semver compatible with Bevy are next +/// - Assets that have more stars are first /// - If all else is equal, sort randomly fn sort_section(nodes: &mut [AssetNode], latest_bevy_version: &semver::Version) { for node in nodes.iter_mut() { @@ -73,18 +75,26 @@ fn sort_section(nodes: &mut [AssetNode], latest_bevy_version: &semver::Version) for node in nodes { let is_semver_compat = node_semver_compat_with(node, latest_bevy_version); - let existing_order = match node { - AssetNode::Asset(asset) => asset.order.unwrap_or(usize::MAX), + let (existing_order, stars) = match node { + AssetNode::Asset(asset) => { + (asset.order.unwrap_or(usize::MAX), asset.stars.unwrap_or(0)) + } _ => continue, }; let random: u32 = rand::random(); - to_sort.push((node, existing_order, !is_semver_compat, random)); + to_sort.push(( + node, + existing_order, + !is_semver_compat, + -(stars as i32), + random, + )); } - to_sort.sort_by_key(|sorts| (sorts.1, sorts.2, sorts.3)); + to_sort.sort_by_key(|sorts| (sorts.1, sorts.2, sorts.3, sorts.4)); - for (i, (node, _, _, _)) in to_sort.into_iter().enumerate() { + for (i, (node, ..)) in to_sort.into_iter().enumerate() { let AssetNode::Asset(asset) = node else { continue; }; @@ -128,6 +138,7 @@ struct FrontMatterAssetExtra { licenses: Option>, bevy_versions: Option>, nsfw: Option, + stars: Option, } impl From<&Asset> for FrontMatterAsset { @@ -142,6 +153,7 @@ impl From<&Asset> for FrontMatterAsset { licenses: asset.licenses.clone(), bevy_versions: asset.bevy_versions.clone(), nsfw: asset.nsfw, + stars: asset.stars, }, } } diff --git a/generate-assets/src/github_client.rs b/generate-assets/src/github_client.rs index d7c7c0dd51..d412f67f50 100644 --- a/generate-assets/src/github_client.rs +++ b/generate-assets/src/github_client.rs @@ -14,6 +14,12 @@ struct GithubLicenseResponse { license: GithubLicenseLicense, } +#[derive(Deserialize)] +struct GithubStarsResponse { + #[serde(rename = "stargazers_count")] + stars: u32, +} + #[derive(Deserialize)] struct GithubLicenseLicense { spdx_id: String, @@ -96,6 +102,22 @@ impl GithubClient { } } + /// Gets the star count from a github repo link + /// + /// Note that this method requests the whole repository info + /// If any other fields from it are needed, this method may be modified as to not make unnecessary requests + pub fn get_stars_from_url(&self, link: &str) -> anyhow::Result { + let response: GithubStarsResponse = self + .agent + .get(&format!("{BASE_URL}/repos/{link}")) + .set("Accept", "application/json") + .set("Authorization", &format!("Bearer {}", self.token)) + .call()? + .into_json()?; + + Ok(response.stars) + } + /// Search file by name pub fn search_file( &self, diff --git a/generate-assets/src/gitlab_client.rs b/generate-assets/src/gitlab_client.rs index 7535718b5d..10a5817f4f 100644 --- a/generate-assets/src/gitlab_client.rs +++ b/generate-assets/src/gitlab_client.rs @@ -15,6 +15,12 @@ struct GitlabContentResponse { content: String, } +#[derive(Deserialize)] +struct GitlabStarsResponse { + #[serde(rename = "star_count")] + stars: u32, +} + pub struct GitlabClient { agent: ureq::Agent, // This is not currently used because we have so few assets using gitlab that we don't need it. @@ -76,4 +82,20 @@ impl GitlabClient { bail!("Content is not in base64"); } } + + /// Gets the star count of a gitlab project from the url + /// + /// Note that this requests the whole project info + /// So if any more fields from it are needed, this method may be modified as to not make unnecessary requests + pub fn get_stars_from_url(&self, url: &str) -> anyhow::Result { + let response: GitlabStarsResponse = self + .agent + .get(&format!("{BASE_URL}/{url}")) + .set("Accept", "application/json") + // .set("Authorization", &format!("Bearer {}", self.token)) + .call()? + .into_json()?; + + Ok(response.stars) + } } diff --git a/generate-assets/src/lib.rs b/generate-assets/src/lib.rs index 225c09c152..f439aeb4e9 100644 --- a/generate-assets/src/lib.rs +++ b/generate-assets/src/lib.rs @@ -3,6 +3,7 @@ use cratesio_dbdump_csvtab::rusqlite; use cratesio_dbdump_csvtab::CratesIODumpLoader; use github_client::GithubClient; use gitlab_client::GitlabClient; +use regex::Regex; use serde::Deserialize; use std::cmp::Ordering; use std::{fs, path::PathBuf, str::FromStr}; @@ -29,6 +30,7 @@ pub struct Asset { pub licenses: Option>, pub bevy_versions: Option>, pub nsfw: Option, + pub stars: Option, // this field is not read from the toml file #[serde(skip)] @@ -93,7 +95,7 @@ impl AssetNode { } #[derive(Default)] -/// Where to find metadata (bevy version and license) for assets. +/// Where to find metadata (bevy version and license) and stars for assets. pub struct MetadataSource<'a> { /// Connection to the crates.io database sqlite dump. pub crates_io_db: Option<&'a CratesIoDb>, @@ -108,6 +110,11 @@ pub struct MetadataSource<'a> { /// Initialized with [`get_metadata_from_cratesio_statement`] at the beginning /// of the algorithm, used by [`get_metadata_from_cratesio`] for each asset. pub get_metadata_from_cratesio_statement: Option>, + /// Prepared statement to retrieve the repository link from crates.io. + /// + /// Initialized with [`get_repo_from_cratesio_statement`] at the beginning + /// of the algorithm, used by [`get_repo_stars_from_crates_db`] for each asset. + pub get_repo_from_cratesio_statement: Option>, } /// Entry point the algorithm to find [`Asset`] files inside [`Section`] folders, @@ -139,6 +146,8 @@ pub fn parse_assets( }; metadata_source.get_metadata_from_cratesio_statement = Some(get_metadata_from_cratesio_statement(db, bevy_crates_ids)?); + metadata_source.get_repo_from_cratesio_statement = + Some(get_repo_from_cratesio_statement(db)?); } visit_dirs( @@ -212,6 +221,10 @@ fn visit_dirs( eprintln!("ERROR: {err:?}"); } + if let Err(err) = get_stars(&mut asset, metadata_source) { + eprintln!("Failed to get stars for {}: {err:?}", asset.name); + } + section.content.push(AssetNode::Asset(asset)); } } @@ -236,7 +249,8 @@ fn get_extra_metadata( Some("crates.io") => { if let Some(ref mut statement) = metadata_source.get_metadata_from_cratesio_statement { let crate_name = segments[1]; - Some(get_metadata_from_crates_db(crate_name, statement)?) + let metadata = get_metadata_from_crates_db(crate_name, statement)?; + Some(metadata) } else { None } @@ -436,6 +450,7 @@ fn get_metadata_from_gitlab( .context("Failed to get Cargo.toml from gitlab")?; let cargo_manifest = toml::from_str::(&content)?; + Ok(( get_license(&cargo_manifest), get_bevy_version_from_manifest(&cargo_manifest, bevy_crates), @@ -573,7 +588,7 @@ fn get_bevy_manifest_dependency_version(dep: &cargo_toml::Dependency) -> Option< } /// Downloads the crates.io database dump and open a connection to the db. -pub fn prepare_crates_db() -> anyhow::Result { +pub fn prepare_crates_db(should_update: bool) -> anyhow::Result { let cache_dir = { let mut current_dir = std::env::current_dir()?; current_dir.push("data"); @@ -586,11 +601,18 @@ pub fn prepare_crates_db() -> anyhow::Result { println!("Downloading crates.io data dump"); } - let db = CratesIODumpLoader::default() + let mut db = CratesIODumpLoader::default(); + let mut db = db .tables(&["crates", "dependencies", "versions"]) - .preload(true) - .update()? - .open_db()?; + .preload(true); + + if should_update { + db = db.update()?; + } else { + println!("Skipping `db.update()`"); + } + + let db = db.open_db()?; db.execute_batch( "\ @@ -647,6 +669,157 @@ fn get_metadata_from_crates_db_by_name( } } +/// Gets the star count for an asset from the linked GitHub or GitLab repository, or the repository linked on the crates.io page +fn get_stars(asset: &mut Asset, source: &mut MetadataSource) -> anyhow::Result<()> { + let url = Url::parse(&asset.link)?; + let url_stars = get_stars_from_url(url, source); + if let Ok(stars) = url_stars { + asset.stars = Some(stars); + return Ok(()); + }; + + if let Some(crate_name) = &asset.crate_name { + let Some(repo_statement) = &mut source.get_repo_from_cratesio_statement else { + bail!("Can't get stars from crates.io repository - no get repo from crates.io statement provided"); + }; + + asset.stars = Some(get_repo_stars_from_crates_db( + crate_name, + repo_statement, + source.github_client, + source.gitlab_client, + )?); + Ok(()) + } else { + // Return `url_stars`s error + url_stars.map(|_| ()) + } +} + +/// Gets the star count from a url and [`MetadataSource`] +fn get_stars_from_url(url: Url, metadata_source: &mut MetadataSource) -> anyhow::Result { + let segments = url.path_segments().map(|c| c.collect::>()).unwrap(); + + let stars = match url.host_str() { + Some("crates.io") => { + let Some(ref mut statement) = metadata_source.get_repo_from_cratesio_statement else { + bail!("Can't get stars from crates.io repository - no get repo from crates.io statement provided"); + }; + + let crate_name = segments[1]; + + get_repo_stars_from_crates_db( + crate_name, + statement, + metadata_source.github_client, + metadata_source.gitlab_client, + ) + .unwrap_or_default() + } + Some("github.com") | Some("gitlab.com") => get_stars_from_repo_url( + metadata_source.github_client, + metadata_source.gitlab_client, + url, + )?, + Some(_) => bail!("Unknown host: {url}"), + None => bail!("No host"), + }; + + Ok(stars) +} + +/// Gets the stars of a crate from the crates.io database dump. +/// +/// If the crate is not found, retries with `-` instead of `_` +fn get_repo_stars_from_crates_db( + crate_name: &str, + repo_statement: &mut rusqlite::Statement<'_>, + github_client: Option<&GithubClient>, + gitlab_client: Option<&GitlabClient>, +) -> anyhow::Result { + let url = get_repo_url_from_crates_db(crate_name, repo_statement)?; + + get_stars_from_repo_url(github_client, gitlab_client, url) +} + +/// Gets the star count from a url to a repository +fn get_stars_from_repo_url( + github_client: Option<&GithubClient>, + gitlab_client: Option<&GitlabClient>, + url: Url, +) -> anyhow::Result { + let stars = match url.host_str() { + Some("github.com") => { + let Some(client) = github_client else { + bail!("Can't get stars from GitHub - no GitHub client"); + }; + let mut path = url.path().trim_matches('/').trim_end_matches(".git"); + + let replaced = Regex::new(r#"(.*)\/tree\/.*"#) + .unwrap() + .replace(path, r#"$1"#); + + path = replaced.as_ref(); + + client.get_stars_from_url(path)? + } + Some("gitlab.com") => { + let Some(client) = gitlab_client else { + bail!("Can't get stars from GitLab - no GitLab client") + }; + + // `gitlab.com/api/v4/projects/me/project` 404s, but `gitlab.com/api/v4/project/me%2Fproject` doesn't. + let path = url.path().trim_matches('/').replace("/", "%2F"); + client.get_stars_from_url(&path)? + } + Some(_) => bail!("Unknown repository host: {url}"), + _ => bail!("No repository host"), + }; + + Ok(stars) +} + +/// Gets the repository link of a crate from the crates.io database dump. +/// +/// If the crate is not found, retries with `-` instead of `_` +fn get_repo_url_from_crates_db( + crate_name: &str, + statement: &mut rusqlite::Statement<'_>, +) -> anyhow::Result { + let link: anyhow::Result> = + if let Ok(link) = get_repo_url_from_crates_db_by_name(crate_name, statement) { + Ok(link) + } else if let Ok(link) = + get_repo_url_from_crates_db_by_name(&crate_name.replace('_', "-"), statement) + { + Ok(link) + } else { + bail!("Failed to get repository link from crates.io db"); + }; + + if let Some(link) = link? { + Ok(link) + } else { + bail!("No repository link on crates.io"); + } +} + +/// Gets the repository link of a crate from the crates.io database dump by the exact name. +fn get_repo_url_from_crates_db_by_name( + name: &str, + statement: &mut rusqlite::Statement<'_>, +) -> anyhow::Result> { + let row = statement.query_row([name], |row| Ok(row.get_unwrap::<_, Option>(0)))?; + let Some(row) = row else { + return Ok(None); + }; + if row.is_empty() { + return Ok(None); + } + + Ok(Some(Url::parse(&row)?)) +} + /// Gets at list of the official bevy crates from the crates.io database dump, /// in lexicographic order. fn get_official_bevy_crates_from_crates_io_db( @@ -720,6 +893,21 @@ pub fn get_latest_bevy_version(db: &CratesIoDb) -> anyhow::Result Result, rusqlite::Error> { + db.prepare( + "\ + SELECT repository \ + FROM crates \ + WHERE name = ? \ + ", + ) +} + /// Get a prepared statement to get license and version for a crate from the /// crates.io database dump. /// diff --git a/sass/components/_asset-card.scss b/sass/components/_asset-card.scss index 6de0bc80b9..73bcd97b62 100644 --- a/sass/components/_asset-card.scss +++ b/sass/components/_asset-card.scss @@ -10,6 +10,7 @@ $asset-card-padding: 0.4rem; "desc" "tags"; color: var(--asset-card-text-color); + height: 100%; &__banner { grid-area: banner; @@ -62,7 +63,7 @@ $asset-card-padding: 0.4rem; padding: $asset-card-padding; font-size: 1rem; overflow: ellipsis; - height: 4.2rem; + max-height: 4.2rem; } &__small_description { @@ -81,9 +82,9 @@ $asset-card-padding: 0.4rem; &__tags { grid-area: tags; display: grid; - grid-template-areas: "versions licenses"; + grid-template-areas: "versions right"; padding: $asset-card-padding; - height: 2.2rem; + align-items: end; } &__tag-icon { @@ -101,12 +102,6 @@ $asset-card-padding: 0.4rem; grid-area: versions; } - &__licenses { - grid-area: licenses; - margin-left: auto; - order: 2; - } - &__tag-list { display: flex; align-items: center; @@ -123,4 +118,14 @@ $asset-card-padding: 0.4rem; color: var(--asset-card-tag-text-color); background: var(--asset-card-tag-bg-color); } + + &__tag-right { + display: flex; + align-items: end; + flex-direction: column; + width: fit-content; + margin-left: auto; + grid-area: right; + gap: 0.2em; + } } diff --git a/templates/macros/assets.html b/templates/macros/assets.html index 2d9bef9409..da411a59d1 100644 --- a/templates/macros/assets.html +++ b/templates/macros/assets.html @@ -2,12 +2,17 @@ {% macro init_svg() %} {% endmacro init_svg %} {% macro card(post) %} @@ -38,16 +43,28 @@ {% for version in post.extra.bevy_versions %}{{ version }}{% endfor %} {% endif %} - {% if post.extra.licenses %} -
-
+
+ {% if post.extra.stars %} +
+
- +
- {% for license in post.extra.licenses %}{{ license }}{% endfor %} + {{ post.extra.stars }}
- {% endif %} + {% endif %} + {% if post.extra.licenses %} +
+
+ + + +
+ {% for license in post.extra.licenses %}{{ license }}{% endfor %} +
+ {% endif %} +