From 6102d9008f01234c1d126bc9b9249f0ca51e1073 Mon Sep 17 00:00:00 2001 From: laruh Date: Mon, 28 Aug 2023 18:34:46 +0700 Subject: [PATCH 01/39] impl functions to update spam in native, unit tests added. --- mm2src/coins/nft.rs | 101 ++++++++++++- mm2src/coins/nft/nft_errors.rs | 43 +++++- mm2src/coins/nft/nft_structs.rs | 81 +++++++++- mm2src/coins/nft/nft_tests.rs | 110 +++++++++++++- mm2src/coins/nft/storage/db_test_helpers.rs | 64 +++++++- mm2src/coins/nft/storage/mod.rs | 26 ++++ mm2src/coins/nft/storage/sql_storage.rs | 140 +++++++++++++++++- mm2src/coins/nft/storage/wasm/wasm_storage.rs | 34 +++++ 8 files changed, 575 insertions(+), 24 deletions(-) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index e679947517..81162093c1 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -15,9 +15,9 @@ use nft_structs::{Chain, ContractType, ConvertChain, Nft, NftFromMoralis, NftLis TransactionNftDetails, UpdateNftReq, WithdrawNftReq}; use crate::eth::{eth_addr_to_hex, get_eth_address, withdraw_erc1155, withdraw_erc721}; -use crate::nft::nft_errors::ProtectFromSpamError; -use crate::nft::nft_structs::{NftCommon, NftCtx, NftTransferCommon, RefreshMetadataReq, TransferMeta, TransferStatus, - UriMeta}; +use crate::nft::nft_errors::{ProtectFromSpamError, UpdateSpamPhishingError}; +use crate::nft::nft_structs::{MnemonicHQRes, NftCommon, NftCtx, NftTransferCommon, RefreshMetadataReq, TransferMeta, + TransferStatus, UriMeta}; use crate::nft::storage::{NftListStorageOps, NftStorageBuilder, NftTransferHistoryStorageOps}; use common::{parse_rfc3339_to_timestamp, APPLICATION_JSON}; use ethereum_types::Address; @@ -29,6 +29,12 @@ use serde_json::Value as Json; use std::cmp::Ordering; use std::str::FromStr; +#[cfg(not(target_arch = "wasm32"))] +use mm2_net::native_http::slurp_post_json; + +#[cfg(target_arch = "wasm32")] +use mm2_net::wasm_http::slurp_post_json; + const MORALIS_API_ENDPOINT: &str = "api/v2"; /// query parameters for moralis request: The format of the token ID const MORALIS_FORMAT_QUERY_NAME: &str = "format"; @@ -36,6 +42,13 @@ const MORALIS_FORMAT_QUERY_VALUE: &str = "decimal"; /// The minimum block number from which to get the transfers const MORALIS_FROM_BLOCK_QUERY_NAME: &str = "from_block"; +const BLOCKLIST_ENDPOINT: &str = "api/blocklist"; +const BLOCKLIST_CONTRACT: &str = "contract"; +#[allow(dead_code)] +const BLOCKLIST_DOMAIN: &str = "domain"; +const BLOCKLIST_WALLET: &str = "wallet"; +const BLOCKLIST_SCAN: &str = "scan"; + pub type WithdrawNftResult = Result>; /// `get_nft_list` function returns list of NFTs on requested chains owned by user. @@ -167,10 +180,77 @@ pub async fn update_nft(ctx: MmArc, req: UpdateNftReq) -> MmResult<(), UpdateNft } update_nft_list(ctx.clone(), &storage, chain, scanned_block + 1, &req.url).await?; update_transfers_with_empty_meta(&storage, chain, &req.url).await?; + update_spam_phishing(&ctx, &storage, *chain, &req.url_antispam).await?; } Ok(()) } +#[allow(dead_code)] +/// `update_spam_phishing` function updates spam contracts and phishing domains info in NFT list and NFT transfers. +async fn update_spam_phishing( + ctx: &MmArc, + storage: &T, + chain: Chain, + url_antispam: &Url, +) -> MmResult<(), UpdateSpamPhishingError> +where + T: NftListStorageOps + NftTransferHistoryStorageOps, +{ + if chain == Chain::Eth || chain == Chain::Polygon { + update_spam_nft_with_mnemonichq(ctx, storage, &chain, url_antispam).await?; + } + let _scan_contract_uri = prepare_uri_for_blocklist_endpoint(url_antispam, BLOCKLIST_CONTRACT, BLOCKLIST_SCAN)?; + // todo get all unique contracts from transfer table (as it also covers NFT list table) and send post req with these addresses. + // todo most likely `update_spam_phishing` will be renamed to `update_spam` as there will be too many logic and code with phishing impl. + todo!() +} +/// Uses `/api/blocklist/wallet/{network}/{wallet_address}` endpoint to get **all spam contract addresses** of NFTs owned by the user. +/// Currently, only the `Eth` and `Polygon` networks are supported. +async fn update_spam_nft_with_mnemonichq( + ctx: &MmArc, + storage: &T, + chain: &Chain, + url_antispam: &Url, +) -> MmResult<(), UpdateSpamPhishingError> +where + T: NftListStorageOps + NftTransferHistoryStorageOps, +{ + let mut scan_wallet_uri = prepare_uri_for_blocklist_endpoint(url_antispam, BLOCKLIST_WALLET, &chain.to_string())?; + let req = MyAddressReq { + coin: chain.to_ticker(), + }; + let my_address = get_my_address(ctx.clone(), req).await?.wallet_address.to_lowercase(); + scan_wallet_uri + .path_segments_mut() + .map_to_mm(|_| UpdateSpamPhishingError::Internal("Invalid URI".to_string()))? + .push(&my_address); + let response = send_request_to_uri(scan_wallet_uri.as_str()).await?; + let mnemonichq_res: MnemonicHQRes = serde_json::from_value(response)?; + for contract in mnemonichq_res.spam_contracts.iter() { + storage + .update_nft_spam_by_token_address(chain, eth_addr_to_hex(contract), true) + .await?; + storage + .update_transfer_spam_by_token_address(chain, eth_addr_to_hex(contract), true) + .await?; + } + Ok(()) +} + +fn prepare_uri_for_blocklist_endpoint( + url_antispam: &Url, + blocklist_type: &str, + blocklist_action_or_network: &str, +) -> MmResult { + let mut uri = url_antispam.clone(); + uri.set_path(BLOCKLIST_ENDPOINT); + uri.path_segments_mut() + .map_to_mm(|_| UpdateSpamPhishingError::Internal("Invalid URI".to_string()))? + .push(blocklist_type) + .push(blocklist_action_or_network); + Ok(uri) +} + pub async fn refresh_nft_metadata(ctx: MmArc, req: RefreshMetadataReq) -> MmResult<(), UpdateNftError> { let nft_ctx = NftCtx::from_ctx(&ctx).map_to_mm(GetNftInfoError::Internal)?; let _lock = nft_ctx.guard.lock().await; @@ -318,6 +398,7 @@ async fn get_moralis_nft_transfers( verified: transfer_moralis.common.verified, operator: transfer_moralis.common.operator, possible_spam: transfer_moralis.common.possible_spam, + possible_phishing: false, }, chain: *chain, block_number: *transfer_moralis.block_number, @@ -408,6 +489,18 @@ async fn send_request_to_uri(uri: &str) -> MmResult { Ok(body) } +#[allow(dead_code)] +async fn send_post_request_to_uri(uri: &str, body: String) -> MmResult, GetInfoFromUriError> { + let (status, _header, body) = slurp_post_json(uri, body).await?; + if !status.is_success() { + return Err(MmError::new(GetInfoFromUriError::Transport(format!( + "Response !200 from {}: {}", + uri, status, + )))); + } + Ok(body) +} + #[cfg(target_arch = "wasm32")] async fn send_request_to_uri(uri: &str) -> MmResult { use mm2_net::wasm_http::FetchRequest; @@ -719,6 +812,7 @@ async fn handle_receive_erc1155 for UpdateNftError { @@ -190,6 +191,10 @@ impl From for UpdateNftError { fn from(err: T) -> Self { UpdateNftError::DbError(format!("{:?}", err)) } } +impl From for UpdateNftError { + fn from(e: UpdateSpamPhishingError) -> Self { UpdateNftError::UpdateSpamPhishingError(e) } +} + impl HttpStatusCode for UpdateNftError { fn status_code(&self) -> StatusCode { match self { @@ -202,7 +207,8 @@ impl HttpStatusCode for UpdateNftError { | UpdateNftError::InvalidBlockOrder { .. } | UpdateNftError::LastScannedBlockNotFound { .. } | UpdateNftError::AttemptToReceiveAlreadyOwnedErc721 { .. } - | UpdateNftError::InvalidHexString(_) => StatusCode::INTERNAL_SERVER_ERROR, + | UpdateNftError::InvalidHexString(_) + | UpdateNftError::UpdateSpamPhishingError(_) => StatusCode::INTERNAL_SERVER_ERROR, } } } @@ -241,3 +247,38 @@ pub enum ProtectFromSpamError { #[from_stringify("serde_json::Error")] SerdeError(String), } + +#[derive(Clone, Debug, Deserialize, Display, EnumFromStringify, PartialEq, Serialize)] +pub enum UpdateSpamPhishingError { + #[display(fmt = "Invalid request: {}", _0)] + InvalidRequest(String), + #[display(fmt = "Transport: {}", _0)] + Transport(String), + #[from_stringify("serde_json::Error")] + #[display(fmt = "Invalid response: {}", _0)] + InvalidResponse(String), + #[display(fmt = "Internal: {}", _0)] + Internal(String), + #[display(fmt = "DB error {}", _0)] + DbError(String), + GetMyAddressError(GetMyAddressError), +} + +impl From for UpdateSpamPhishingError { + fn from(e: GetMyAddressError) -> Self { UpdateSpamPhishingError::GetMyAddressError(e) } +} + +impl From for UpdateSpamPhishingError { + fn from(e: GetInfoFromUriError) -> Self { + match e { + GetInfoFromUriError::InvalidRequest(e) => UpdateSpamPhishingError::InvalidRequest(e), + GetInfoFromUriError::Transport(e) => UpdateSpamPhishingError::Transport(e), + GetInfoFromUriError::InvalidResponse(e) => UpdateSpamPhishingError::InvalidResponse(e), + GetInfoFromUriError::Internal(e) => UpdateSpamPhishingError::Internal(e), + } + } +} + +impl From for UpdateSpamPhishingError { + fn from(err: T) -> Self { UpdateSpamPhishingError::DbError(format!("{:?}", err)) } +} diff --git a/mm2src/coins/nft/nft_structs.rs b/mm2src/coins/nft/nft_structs.rs index 9d7a07b89a..4b13ed48bc 100644 --- a/mm2src/coins/nft/nft_structs.rs +++ b/mm2src/coins/nft/nft_structs.rs @@ -6,8 +6,10 @@ use futures::lock::Mutex as AsyncMutex; use mm2_core::mm_ctx::{from_ctx, MmArc}; use mm2_number::BigDecimal; use rpc::v1::types::Bytes as BytesJson; +use serde::de::{self, Deserializer}; use serde::Deserialize; use serde_json::Value as Json; +use std::collections::HashMap; use std::fmt; use std::num::NonZeroUsize; use std::str::FromStr; @@ -20,6 +22,7 @@ use mm2_db::indexed_db::{ConstructibleDb, SharedDb}; #[cfg(target_arch = "wasm32")] use crate::nft::storage::wasm::nft_idb::NftCacheIDB; +#[allow(dead_code)] #[derive(Debug, Deserialize)] pub struct NftListReq { pub(crate) chains: Vec, @@ -30,6 +33,16 @@ pub struct NftListReq { pub(crate) page_number: Option, #[serde(default)] pub(crate) protect_from_spam: bool, + pub(crate) filters: Option, +} + +#[allow(dead_code)] +#[derive(Copy, Clone, Debug, Deserialize)] +pub struct NftListFilters { + #[serde(default)] + pub(crate) exclude_spam: bool, + #[serde(default)] + pub(crate) exclude_phishing: bool, } #[derive(Debug, Deserialize)] @@ -54,7 +67,7 @@ pub enum ParseChainTypeError { UnsupportedChainType, } -#[derive(Clone, Copy, Debug, Deserialize, Serialize)] +#[derive(Clone, Copy, Debug, PartialEq, Serialize)] #[serde(rename_all = "UPPERCASE")] pub enum Chain { Avalanche, @@ -99,15 +112,31 @@ impl FromStr for Chain { fn from_str(s: &str) -> Result { match s { "AVALANCHE" => Ok(Chain::Avalanche), + "avalanche" => Ok(Chain::Avalanche), "BSC" => Ok(Chain::Bsc), + "bsc" => Ok(Chain::Bsc), "ETH" => Ok(Chain::Eth), + "eth" => Ok(Chain::Eth), "FANTOM" => Ok(Chain::Fantom), + "fantom" => Ok(Chain::Fantom), "POLYGON" => Ok(Chain::Polygon), + "polygon" => Ok(Chain::Polygon), _ => Err(ParseChainTypeError::UnsupportedChainType), } } } +/// This implementation will use `FromStr` to deserialize `Chain`. +impl<'de> Deserialize<'de> for Chain { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let s = String::deserialize(deserializer)?; + s.parse().map_err(de::Error::custom) + } +} + #[derive(Debug, Display)] pub(crate) enum ParseContractTypeError { UnsupportedContractType, @@ -195,6 +224,7 @@ impl UriMeta { } } +#[allow(dead_code)] /// [`NftCommon`] structure contains common fields from [`Nft`] and [`NftFromMoralis`] #[derive(Clone, Debug, Deserialize, Serialize)] pub struct NftCommon { @@ -213,6 +243,8 @@ pub struct NftCommon { pub(crate) minter_address: Option, #[serde(default)] pub(crate) possible_spam: bool, + #[serde(default)] + pub(crate) possible_phishing: bool, } #[derive(Clone, Debug, Deserialize, Serialize)] @@ -322,6 +354,7 @@ pub struct TransactionNftDetails { pub(crate) transaction_type: TransactionType, } +#[allow(dead_code)] #[derive(Debug, Deserialize)] pub struct NftTransfersReq { pub(crate) chains: Vec, @@ -340,7 +373,7 @@ pub(crate) enum ParseTransferStatusError { UnsupportedTransferStatus, } -#[derive(Debug, Deserialize, Clone, Copy, PartialEq, Serialize)] +#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Serialize)] pub(crate) enum TransferStatus { Receive, Send, @@ -368,6 +401,7 @@ impl fmt::Display for TransferStatus { } } +#[allow(dead_code)] /// [`NftTransferCommon`] structure contains common fields from [`NftTransferHistory`] and [`NftTransferHistoryFromMoralis`] #[derive(Clone, Debug, Deserialize, Serialize)] pub struct NftTransferCommon { @@ -387,6 +421,8 @@ pub struct NftTransferCommon { pub(crate) operator: Option, #[serde(default)] pub(crate) possible_spam: bool, + #[serde(default)] + pub(crate) possible_phishing: bool, } #[derive(Clone, Debug, Deserialize, Serialize)] @@ -421,20 +457,27 @@ pub struct NftsTransferHistoryList { pub(crate) total: usize, } +#[allow(dead_code)] #[derive(Copy, Clone, Debug, Deserialize)] pub struct NftTransferHistoryFilters { #[serde(default)] - pub receive: bool, + pub(crate) receive: bool, #[serde(default)] pub(crate) send: bool, pub(crate) from_date: Option, pub(crate) to_date: Option, + #[serde(default)] + pub(crate) exclude_spam: bool, + #[serde(default)] + pub(crate) exclude_phishing: bool, } +#[allow(dead_code)] #[derive(Debug, Deserialize)] pub struct UpdateNftReq { pub(crate) chains: Vec, pub(crate) url: Url, + pub(crate) url_antispam: Url, } #[derive(Debug, Deserialize, Eq, Hash, PartialEq)] @@ -483,3 +526,35 @@ impl NftCtx { }))) } } + +#[derive(Debug, Serialize)] +pub(crate) struct SpamContractReq { + pub(crate) network: Chain, + pub(crate) addresses: String, +} + +#[derive(Debug, Serialize)] +pub(crate) struct PhishingDomainReq { + pub(crate) domains: String, +} + +#[allow(dead_code)] +#[derive(Debug, Deserialize)] +pub(crate) struct SpamContractRes { + pub(crate) result: HashMap, +} + +#[allow(dead_code)] +#[derive(Debug, Deserialize)] +pub(crate) struct PhishingDomainRes { + pub(crate) result: HashMap, +} + +#[allow(dead_code)] +#[derive(Debug, Deserialize)] +pub(crate) struct MnemonicHQRes { + pub(crate) network: Chain, + pub(crate) address: Address, + pub(crate) result: String, + pub(crate) spam_contracts: Vec
, +} diff --git a/mm2src/coins/nft/nft_tests.rs b/mm2src/coins/nft/nft_tests.rs index 02710e0ac4..d1351d50c1 100644 --- a/mm2src/coins/nft/nft_tests.rs +++ b/mm2src/coins/nft/nft_tests.rs @@ -2,16 +2,25 @@ const NFT_LIST_URL_TEST: &str = "https://moralis-proxy.komodo.earth/api/v2/0x394 const NFT_HISTORY_URL_TEST: &str = "https://moralis-proxy.komodo.earth/api/v2/0x394d86994f954ed931b86791b62fe64f4c5dac37/nft/transfers?chain=POLYGON&format=decimal"; const NFT_METADATA_URL_TEST: &str = "https://moralis-proxy.komodo.earth/api/v2/nft/0xed55e4477b795eaa9bb4bca24df42214e1a05c18/1111777?chain=POLYGON&format=decimal"; const TEST_WALLET_ADDR_EVM: &str = "0x394d86994f954ed931b86791b62fe64f4c5dac37"; +const BLOCKLIST_WALLET_ENDPOINT: &str = + "https://nft.antispam.dragonhound.info/api/blocklist/wallet/eth/0x3eb4b12127EdC81A4d2fD49658db07005bcAd065"; +const BLOCKLIST_CONTRACT_SCAN_ENDPOINT: &str = "https://nft.antispam.dragonhound.info/api/blocklist/contract/scan"; +const BLOCKLIST_DOMAIN_SCAN_ENDPOINT: &str = "https://nft.antispam.dragonhound.info/api/blocklist/domain/scan"; #[cfg(all(test, not(target_arch = "wasm32")))] mod native_tests { use crate::eth::eth_addr_to_hex; - use crate::nft::nft_structs::{NftFromMoralis, NftTransferHistoryFromMoralis, UriMeta}; - use crate::nft::nft_tests::{NFT_HISTORY_URL_TEST, NFT_LIST_URL_TEST, NFT_METADATA_URL_TEST, TEST_WALLET_ADDR_EVM}; + use crate::nft::nft_structs::{Chain, MnemonicHQRes, NftFromMoralis, NftTransferHistoryFromMoralis, + PhishingDomainReq, PhishingDomainRes, SpamContractReq, SpamContractRes, UriMeta}; + use crate::nft::nft_tests::{BLOCKLIST_CONTRACT_SCAN_ENDPOINT, BLOCKLIST_DOMAIN_SCAN_ENDPOINT, + BLOCKLIST_WALLET_ENDPOINT, NFT_HISTORY_URL_TEST, NFT_LIST_URL_TEST, + NFT_METADATA_URL_TEST, TEST_WALLET_ADDR_EVM}; use crate::nft::storage::db_test_helpers::*; use crate::nft::{check_and_redact_if_spam, check_moralis_ipfs_bafy, check_nft_metadata_for_spam, - send_request_to_uri}; + send_post_request_to_uri, send_request_to_uri}; use common::block_on; + use ethereum_types::Address; + use std::str::FromStr; #[test] fn test_moralis_ipfs_bafy() { @@ -89,11 +98,46 @@ mod native_tests { serde_json::from_str::(&uri_response.to_string()).unwrap(); } + #[test] + fn test_antispam_api_requests() { + let mnemonichq_value = block_on(send_request_to_uri(BLOCKLIST_WALLET_ENDPOINT)).unwrap(); + let mnemonichq_res: MnemonicHQRes = serde_json::from_value(mnemonichq_value).unwrap(); + assert!(mnemonichq_res + .spam_contracts + .contains(&Address::from_str("0x0ded8542fc8b2b4e781b96e99fee6406550c9b7c").unwrap())); + assert_eq!(Chain::Eth, mnemonichq_res.network); + + let req_spam = SpamContractReq { + network: Chain::Eth, + addresses: "0x0ded8542fc8b2b4e781b96e99fee6406550c9b7c,0x8d1355b65da254f2cc4611453adfa8b7a13f60ee" + .to_string(), + }; + let req_json = serde_json::to_string(&req_spam).unwrap(); + let contract_scan_res = block_on(send_post_request_to_uri(BLOCKLIST_CONTRACT_SCAN_ENDPOINT, req_json)).unwrap(); + let spam_res: SpamContractRes = serde_json::from_slice(&contract_scan_res).unwrap(); + assert!(spam_res + .result + .get(&Address::from_str("0x0ded8542fc8b2b4e781b96e99fee6406550c9b7c").unwrap()) + .unwrap()); + assert!(spam_res + .result + .get(&Address::from_str("0x8d1355b65da254f2cc4611453adfa8b7a13f60ee").unwrap()) + .unwrap()); + + let req_phishing = PhishingDomainReq { + domains: "disposal-account-case-1f677.web.app,defi8090.vip".to_string(), + }; + let req_json = serde_json::to_string(&req_phishing).unwrap(); + let domain_scan_res = block_on(send_post_request_to_uri(BLOCKLIST_DOMAIN_SCAN_ENDPOINT, req_json)).unwrap(); + let phishing_res: PhishingDomainRes = serde_json::from_slice(&domain_scan_res).unwrap(); + assert!(phishing_res.result.get("disposal-account-case-1f677.web.app").unwrap()); + } + #[test] fn test_add_get_nfts() { block_on(test_add_get_nfts_impl()) } #[test] - fn test_last_nft_blocks() { block_on(test_last_nft_blocks_impl()) } + fn test_last_nft_block() { block_on(test_last_nft_block_impl()) } #[test] fn test_nft_list() { block_on(test_nft_list_impl()) } @@ -104,6 +148,9 @@ mod native_tests { #[test] fn test_refresh_metadata() { block_on(test_refresh_metadata_impl()) } + #[test] + fn test_nft_spam_by_token_address() { block_on(test_update_nft_spam_by_token_address_impl()) } + #[test] fn test_nft_amount() { block_on(test_nft_amount_impl()) } @@ -121,15 +168,23 @@ mod native_tests { #[test] fn test_get_update_transfer_meta() { block_on(test_get_update_transfer_meta_impl()) } + + #[test] + fn test_update_transfer_spam_by_token_address() { block_on(test_update_transfer_spam_by_token_address_impl()) } } #[cfg(target_arch = "wasm32")] mod wasm_tests { use crate::eth::eth_addr_to_hex; - use crate::nft::nft_structs::{NftFromMoralis, NftTransferHistoryFromMoralis}; - use crate::nft::nft_tests::{NFT_HISTORY_URL_TEST, NFT_LIST_URL_TEST, NFT_METADATA_URL_TEST, TEST_WALLET_ADDR_EVM}; - use crate::nft::send_request_to_uri; + use crate::nft::nft_structs::{Chain, MnemonicHQRes, NftFromMoralis, NftTransferHistoryFromMoralis, + PhishingDomainReq, PhishingDomainRes, SpamContractReq, SpamContractRes}; + use crate::nft::nft_tests::{BLOCKLIST_CONTRACT_SCAN_ENDPOINT, BLOCKLIST_DOMAIN_SCAN_ENDPOINT, + BLOCKLIST_WALLET_ENDPOINT, NFT_HISTORY_URL_TEST, NFT_LIST_URL_TEST, + NFT_METADATA_URL_TEST, TEST_WALLET_ADDR_EVM}; use crate::nft::storage::db_test_helpers::*; + use crate::nft::{send_post_request_to_uri, send_request_to_uri}; + use ethereum_types::Address; + use std::str::FromStr; use wasm_bindgen_test::*; wasm_bindgen_test_configure!(run_in_browser); @@ -159,11 +214,50 @@ mod wasm_tests { assert_eq!(41237364, *nft_moralis.block_number_minted.unwrap()); } + #[wasm_bindgen_test] + async fn test_antispam_api_requests() { + let res_value = send_request_to_uri(BLOCKLIST_WALLET_ENDPOINT).await.unwrap(); + let mnemonichq_res: MnemonicHQRes = serde_json::from_value(res_value).unwrap(); + assert!(mnemonichq_res + .spam_contracts + .contains(&Address::from_str("0x0ded8542fc8b2b4e781b96e99fee6406550c9b7c").unwrap())); + assert_eq!(Chain::Eth, mnemonichq_res.network); + + let req_spam = SpamContractReq { + network: Chain::Eth, + addresses: "0x0ded8542fc8b2b4e781b96e99fee6406550c9b7c,0x8d1355b65da254f2cc4611453adfa8b7a13f60ee" + .to_string(), + }; + let req_json = serde_json::to_string(&req_spam).unwrap(); + let contract_scan_res = send_post_request_to_uri(BLOCKLIST_CONTRACT_SCAN_ENDPOINT, req_json) + .await + .unwrap(); + let spam_res: SpamContractRes = serde_json::from_slice(&contract_scan_res).unwrap(); + assert!(spam_res + .result + .get(&Address::from_str("0x0ded8542fc8b2b4e781b96e99fee6406550c9b7c").unwrap()) + .unwrap()); + assert!(spam_res + .result + .get(&Address::from_str("0x8d1355b65da254f2cc4611453adfa8b7a13f60ee").unwrap()) + .unwrap()); + + let req_phishing = PhishingDomainReq { + domains: "disposal-account-case-1f677.web.app,defi8090.vip".to_string(), + }; + let req_json = serde_json::to_string(&req_phishing).unwrap(); + let domain_scan_res = send_post_request_to_uri(BLOCKLIST_DOMAIN_SCAN_ENDPOINT, req_json) + .await + .unwrap(); + let phishing_res: PhishingDomainRes = serde_json::from_slice(&domain_scan_res).unwrap(); + assert!(phishing_res.result.get("disposal-account-case-1f677.web.app").unwrap()); + } + #[wasm_bindgen_test] async fn test_add_get_nfts() { test_add_get_nfts_impl().await } #[wasm_bindgen_test] - async fn test_last_nft_blocks() { test_last_nft_blocks_impl().await } + async fn test_last_nft_block() { test_last_nft_block_impl().await } #[wasm_bindgen_test] async fn test_nft_list() { test_nft_list_impl().await } diff --git a/mm2src/coins/nft/storage/db_test_helpers.rs b/mm2src/coins/nft/storage/db_test_helpers.rs index 7961d4c1ea..a105e48392 100644 --- a/mm2src/coins/nft/storage/db_test_helpers.rs +++ b/mm2src/coins/nft/storage/db_test_helpers.rs @@ -38,6 +38,7 @@ pub(crate) fn nft() -> Nft { last_metadata_sync: Some("2023-02-07T17:10:16.858Z".to_string()), minter_address: Some("ERC1155 tokens don't have a single minter".to_string()), possible_spam: false, + possible_phishing: false, }, chain: Chain::Bsc, block_number_minted: Some(25465916), @@ -74,6 +75,7 @@ fn transfer() -> NftTransferHistory { verified: Some(1), operator: None, possible_spam: false, + possible_phishing: false, }, chain: Chain::Bsc, block_number: 28056726, @@ -103,6 +105,7 @@ fn nft_list() -> Vec { last_metadata_sync: Some("2023-02-07T17:10:16.858Z".to_string()), minter_address: Some("ERC1155 tokens don't have a single minter".to_string()), possible_spam: false, + possible_phishing: false, }, chain: Chain::Bsc, block_number_minted: Some(25465916), @@ -138,6 +141,7 @@ fn nft_list() -> Vec { last_metadata_sync: Some("2023-02-16T16:36:04.283Z".to_string()), minter_address: Some("0xdbdeb0895f3681b87fb3654b5cf3e05546ba24a9".to_string()), possible_spam: false, + possible_phishing: false, }, chain: Chain::Bsc, @@ -176,6 +180,7 @@ fn nft_list() -> Vec { last_metadata_sync: Some("2023-02-16T16:36:04.283Z".to_string()), minter_address: Some("0xdbdeb0895f3681b87fb3654b5cf3e05546ba24a9".to_string()), possible_spam: false, + possible_phishing: false, }, chain: Chain::Bsc, @@ -214,6 +219,7 @@ fn nft_list() -> Vec { last_metadata_sync: Some("2023-02-19T19:12:18.080Z".to_string()), minter_address: Some("0xdbdeb0895f3681b87fb3654b5cf3e05546ba24a9".to_string()), possible_spam: false, + possible_phishing: false, }, chain: Chain::Bsc, @@ -253,6 +259,7 @@ fn nft_transfer_history() -> Vec { verified: Some(1), operator: Some("0x4ff0bbc9b64d635a4696d1a38554fb2529c103ff".to_string()), possible_spam: false, + possible_phishing: false, }, chain: Chain::Bsc, block_number: 25919780, @@ -281,6 +288,7 @@ fn nft_transfer_history() -> Vec { verified: Some(1), operator: None, possible_spam: false, + possible_phishing: false, }, chain: Chain::Bsc, block_number: 28056726, @@ -312,6 +320,7 @@ fn nft_transfer_history() -> Vec { verified: Some(1), operator: None, possible_spam: false, + possible_phishing: false, }, chain: Chain::Bsc, block_number: 28056726, @@ -342,6 +351,7 @@ fn nft_transfer_history() -> Vec { verified: Some(1), operator: None, possible_spam: false, + possible_phishing: false, }, chain: Chain::Bsc, block_number: 28056721, @@ -394,19 +404,17 @@ pub(crate) async fn test_add_get_nfts_impl() { assert_eq!(nft.block_number, 28056721); } -pub(crate) async fn test_last_nft_blocks_impl() { +pub(crate) async fn test_last_nft_block_impl() { let chain = Chain::Bsc; let storage = init_nft_list_storage(&chain).await; let nft_list = nft_list(); storage.add_nfts_to_list(&chain, nft_list, 28056726).await.unwrap(); - let token_id = BigDecimal::from_str(TOKEN_ID).unwrap(); - let nft = storage - .get_nft(&chain, TOKEN_ADD.to_string(), token_id) + let last_block = NftListStorageOps::get_last_block_number(&storage, &chain) .await .unwrap() .unwrap(); - assert_eq!(nft.block_number, 28056721); + assert_eq!(last_block, 28056726); } pub(crate) async fn test_nft_list_impl() { @@ -507,6 +515,26 @@ pub(crate) async fn test_refresh_metadata_impl() { assert_eq!(new_symbol.to_string(), nft_upd.common.symbol.unwrap()); } +#[allow(dead_code)] +pub(crate) async fn test_update_nft_spam_by_token_address_impl() { + let chain = Chain::Bsc; + let storage = init_nft_list_storage(&chain).await; + let nft_list = nft_list(); + storage.add_nfts_to_list(&chain, nft_list, 28056726).await.unwrap(); + + storage + .update_nft_spam_by_token_address(&chain, TOKEN_ADD.to_string(), true) + .await + .unwrap(); + let nfts = storage + .get_nfts_by_token_address(&chain, TOKEN_ADD.to_string()) + .await + .unwrap(); + for nft in nfts { + assert!(nft.common.possible_spam); + } +} + pub(crate) async fn test_add_get_transfers_impl() { let chain = Chain::Bsc; let storage = init_nft_history_storage(&chain).await; @@ -573,6 +601,8 @@ pub(crate) async fn test_transfer_history_filters_impl() { send: false, from_date: None, to_date: None, + exclude_spam: false, + exclude_phishing: false, }; let filters1 = NftTransferHistoryFilters { @@ -580,6 +610,8 @@ pub(crate) async fn test_transfer_history_filters_impl() { send: false, from_date: None, to_date: Some(1677166110), + exclude_spam: false, + exclude_phishing: false, }; let filters2 = NftTransferHistoryFilters { @@ -587,6 +619,8 @@ pub(crate) async fn test_transfer_history_filters_impl() { send: false, from_date: Some(1677166110), to_date: Some(1683627417), + exclude_spam: false, + exclude_phishing: false, }; let transfer_history = storage @@ -657,3 +691,23 @@ pub(crate) async fn test_get_update_transfer_meta_impl() { .unwrap(); assert_eq!(transfer_by_hash.token_name, Some("Nebula Nodes".to_string())) } + +#[allow(dead_code)] +pub(crate) async fn test_update_transfer_spam_by_token_address_impl() { + let chain = Chain::Bsc; + let storage = init_nft_history_storage(&chain).await; + let transfers = nft_transfer_history(); + storage.add_transfers_to_history(&chain, transfers).await.unwrap(); + + storage + .update_transfer_spam_by_token_address(&chain, TOKEN_ADD.to_string(), true) + .await + .unwrap(); + let transfers = storage + .get_transfers_by_token_address(&chain, TOKEN_ADD.to_string()) + .await + .unwrap(); + for transfers in transfers { + assert!(transfers.common.possible_spam); + } +} diff --git a/mm2src/coins/nft/storage/mod.rs b/mm2src/coins/nft/storage/mod.rs index 0a2e906ccc..6868a2ba46 100644 --- a/mm2src/coins/nft/storage/mod.rs +++ b/mm2src/coins/nft/storage/mod.rs @@ -85,6 +85,17 @@ pub trait NftListStorageOps { async fn update_nft_amount(&self, chain: &Chain, nft: Nft, scanned_block: u64) -> MmResult<(), Self::Error>; async fn update_nft_amount_and_block_number(&self, chain: &Chain, nft: Nft) -> MmResult<(), Self::Error>; + + /// `get_nfts_by_token_address` function returns list of NFTs which have specified token address. + async fn get_nfts_by_token_address(&self, chain: &Chain, token_address: String) -> MmResult, Self::Error>; + + /// `update_nft_spam_by_token_address` function updates `possible_spam` field in NFTs which have specified token address. + async fn update_nft_spam_by_token_address( + &self, + chain: &Chain, + token_address: String, + possible_spam: bool, + ) -> MmResult<(), Self::Error>; } #[async_trait] @@ -148,6 +159,21 @@ pub trait NftTransferHistoryStorageOps { ) -> MmResult<(), Self::Error>; async fn get_transfers_with_empty_meta(&self, chain: &Chain) -> MmResult, Self::Error>; + + /// `get_transfers_by_token_address` function returns list of NFT transfers which have specified token address. + async fn get_transfers_by_token_address( + &self, + chain: &Chain, + token_address: String, + ) -> MmResult, Self::Error>; + + /// `update_transfer_spam_by_token_address` function updates `possible_spam` field in NFT transfers which have specified token address. + async fn update_transfer_spam_by_token_address( + &self, + chain: &Chain, + token_address: String, + possible_spam: bool, + ) -> MmResult<(), Self::Error>; } #[derive(Debug, Deserialize, Display, Serialize)] diff --git a/mm2src/coins/nft/storage/sql_storage.rs b/mm2src/coins/nft/storage/sql_storage.rs index 9179704467..0d40fdc029 100644 --- a/mm2src/coins/nft/storage/sql_storage.rs +++ b/mm2src/coins/nft/storage/sql_storage.rs @@ -37,6 +37,8 @@ fn create_nft_list_table_sql(chain: &Chain) -> MmResult { amount VARCHAR(256) NOT NULL, block_number INTEGER NOT NULL, contract_type TEXT NOT NULL, + possible_spam INTEGER DEFAULT 0 NOT NULL, + possible_phishing INTEGER DEFAULT 0 NOT NULL, details_json TEXT, PRIMARY KEY (token_address, token_id) );", @@ -64,6 +66,8 @@ fn create_transfer_history_table_sql(chain: &Chain) -> MmResult MmResult { let sql = format!( "INSERT INTO {} ( - token_address, token_id, chain, amount, block_number, contract_type, details_json + token_address, token_id, chain, amount, block_number, contract_type, possible_spam, + possible_phishing, details_json ) VALUES ( - ?1, ?2, ?3, ?4, ?5, ?6, ?7 + ?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9 );", table_name ); @@ -240,9 +245,10 @@ fn insert_transfer_in_history_sql(chain: &Chain) -> MmResult { let sql = format!( "INSERT INTO {} ( transaction_hash, log_index, chain, block_number, block_timestamp, contract_type, - token_address, token_id, status, amount, collection_name, image_url, token_name, details_json + token_address, token_id, status, amount, token_uri, collection_name, image_url, token_name, + possible_spam, possible_phishing, details_json ) VALUES ( - ?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14 + ?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17 );", table_name ); @@ -369,6 +375,21 @@ fn block_number_from_row(row: &Row<'_>) -> Result { row.get::<_, fn nft_amount_from_row(row: &Row<'_>) -> Result { row.get(0) } +fn get_nfts_by_token_address_statement<'a, F>( + conn: &'a Connection, + chain: &'a Chain, + table_name_creator: F, +) -> MmResult, SqlError> +where + F: FnOnce(&Chain) -> String, +{ + let table_name = table_name_creator(chain); + validate_table_name(table_name.as_str())?; + let sql_query = format!("SELECT details_json FROM {} WHERE token_address = ?", table_name); + let stmt = conn.prepare(&sql_query)?; + Ok(stmt) +} + fn get_transfers_from_block_builder<'a>( conn: &'a Connection, chain: &'a Chain, @@ -521,6 +542,8 @@ impl NftListStorageOps for SqliteNftStorage { Some(nft.common.amount.to_string()), Some(nft.block_number.to_string()), Some(nft.contract_type.to_string()), + Some(i32::from(nft.common.possible_spam).to_string()), + Some(i32::from(nft.common.possible_phishing).to_string()), Some(nft_json), ]; sql_transaction.execute(&insert_nft_in_list_sql(&chain)?, params)?; @@ -683,6 +706,57 @@ impl NftListStorageOps for SqliteNftStorage { }) .await } + + async fn get_nfts_by_token_address(&self, chain: &Chain, token_address: String) -> MmResult, Self::Error> { + let selfi = self.clone(); + let chain = *chain; + async_blocking(move || { + let conn = selfi.0.lock().unwrap(); + let mut stmt = get_nfts_by_token_address_statement(&conn, &chain, nft_list_table_name)?; + let nfts = stmt + .query_map([token_address], nft_from_row)? + .collect::, _>>()?; + Ok(nfts) + }) + .await + } + + async fn update_nft_spam_by_token_address( + &self, + chain: &Chain, + token_address: String, + possible_spam: bool, + ) -> MmResult<(), Self::Error> { + let selfi = self.clone(); + let nfts = selfi.get_nfts_by_token_address(chain, token_address.clone()).await?; + + let table_name = nft_list_table_name(chain); + validate_table_name(&table_name)?; + let sql = format!( + "UPDATE {} SET possible_spam = ?1, details_json = ?2 WHERE token_address = ?3 AND token_id = ?4;", + table_name + ); + + async_blocking(move || { + let mut conn = selfi.0.lock().unwrap(); + let sql_transaction = conn.transaction()?; + + for mut nft in nfts.into_iter() { + nft.common.possible_spam = possible_spam; + let nft_json = json::to_string(&nft).expect("serialization should not fail"); + let params = [ + Some(i32::from(possible_spam).to_string()), + Some(nft_json), + Some(token_address.to_string()), + Some(nft.common.token_id.to_string()), + ]; + sql_transaction.execute(&sql, params)?; + } + sql_transaction.commit()?; + Ok(()) + }) + .await + } } #[async_trait] @@ -774,9 +848,12 @@ impl NftTransferHistoryStorageOps for SqliteNftStorage { Some(transfer.common.token_id.to_string()), Some(transfer.status.to_string()), Some(transfer.common.amount.to_string()), + transfer.token_uri, transfer.collection_name, transfer.image_url, transfer.token_name, + Some(i32::from(transfer.common.possible_spam).to_string()), + Some(i32::from(transfer.common.possible_phishing).to_string()), Some(transfer_json), ]; sql_transaction.execute(&insert_transfer_in_history_sql(&chain)?, params)?; @@ -916,4 +993,59 @@ impl NftTransferHistoryStorageOps for SqliteNftStorage { }) .await } + + async fn get_transfers_by_token_address( + &self, + chain: &Chain, + token_address: String, + ) -> MmResult, Self::Error> { + let selfi = self.clone(); + let chain = *chain; + async_blocking(move || { + let conn = selfi.0.lock().unwrap(); + let mut stmt = get_nfts_by_token_address_statement(&conn, &chain, nft_transfer_history_table_name)?; + let nfts = stmt + .query_map([token_address], transfer_history_from_row)? + .collect::, _>>()?; + Ok(nfts) + }) + .await + } + + async fn update_transfer_spam_by_token_address( + &self, + chain: &Chain, + token_address: String, + possible_spam: bool, + ) -> MmResult<(), Self::Error> { + let selfi = self.clone(); + let transfers = selfi.get_transfers_by_token_address(chain, token_address).await?; + + let table_name = nft_transfer_history_table_name(chain); + validate_table_name(&table_name)?; + let sql = format!( + "UPDATE {} SET possible_spam = ?1, details_json = ?2 WHERE transaction_hash = ?3 AND log_index = ?4;", + table_name + ); + + async_blocking(move || { + let mut conn = selfi.0.lock().unwrap(); + let sql_transaction = conn.transaction()?; + + for mut transfer in transfers.into_iter() { + transfer.common.possible_spam = possible_spam; + let transfer_json = json::to_string(&transfer).expect("serialization should not fail"); + let params = [ + Some(i32::from(possible_spam).to_string()), + Some(transfer_json), + Some(transfer.common.transaction_hash.to_string()), + Some(transfer.common.log_index.to_string()), + ]; + sql_transaction.execute(&sql, params)?; + } + sql_transaction.commit()?; + Ok(()) + }) + .await + } } diff --git a/mm2src/coins/nft/storage/wasm/wasm_storage.rs b/mm2src/coins/nft/storage/wasm/wasm_storage.rs index 058b6cdffd..54e60a2772 100644 --- a/mm2src/coins/nft/storage/wasm/wasm_storage.rs +++ b/mm2src/coins/nft/storage/wasm/wasm_storage.rs @@ -315,6 +315,23 @@ impl NftListStorageOps for IndexedDbNftStorage { .await?; Ok(()) } + + async fn get_nfts_by_token_address( + &self, + _chain: &Chain, + _token_address: String, + ) -> MmResult, Self::Error> { + todo!() + } + + async fn update_nft_spam_by_token_address( + &self, + _chain: &Chain, + _token_address: String, + _possible_spam: bool, + ) -> MmResult<(), Self::Error> { + todo!() + } } #[async_trait] @@ -521,6 +538,23 @@ impl NftTransferHistoryStorageOps for IndexedDbNftStorage { } Ok(res.into_iter().collect()) } + + async fn get_transfers_by_token_address( + &self, + _chain: &Chain, + _token_address: String, + ) -> MmResult, Self::Error> { + todo!() + } + + async fn update_transfer_spam_by_token_address( + &self, + _chain: &Chain, + _token_address: String, + _possible_spam: bool, + ) -> MmResult<(), Self::Error> { + todo!() + } } /// `get_last_block_from_table` function returns the highest block in the table related to certain blockchain type. From d31ca2abbb9129d0d3d575ae94772d29b67f2f56 Mon Sep 17 00:00:00 2001 From: laruh Date: Tue, 29 Aug 2023 21:10:00 +0700 Subject: [PATCH 02/39] update_spam func, db impls for wasm, additional unit tests --- mm2src/coins/nft.rs | 42 +++++-- mm2src/coins/nft/nft_structs.rs | 1 - mm2src/coins/nft/nft_tests.rs | 16 ++- mm2src/coins/nft/storage/db_test_helpers.rs | 12 +- mm2src/coins/nft/storage/mod.rs | 4 + mm2src/coins/nft/storage/sql_storage.rs | 35 ++++++ mm2src/coins/nft/storage/wasm/wasm_storage.rs | 117 +++++++++++++++--- 7 files changed, 195 insertions(+), 32 deletions(-) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index 81162093c1..539b76497e 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -16,8 +16,8 @@ use nft_structs::{Chain, ContractType, ConvertChain, Nft, NftFromMoralis, NftLis use crate::eth::{eth_addr_to_hex, get_eth_address, withdraw_erc1155, withdraw_erc721}; use crate::nft::nft_errors::{ProtectFromSpamError, UpdateSpamPhishingError}; -use crate::nft::nft_structs::{MnemonicHQRes, NftCommon, NftCtx, NftTransferCommon, RefreshMetadataReq, TransferMeta, - TransferStatus, UriMeta}; +use crate::nft::nft_structs::{MnemonicHQRes, NftCommon, NftCtx, NftTransferCommon, RefreshMetadataReq, + SpamContractReq, SpamContractRes, TransferMeta, TransferStatus, UriMeta}; use crate::nft::storage::{NftListStorageOps, NftStorageBuilder, NftTransferHistoryStorageOps}; use common::{parse_rfc3339_to_timestamp, APPLICATION_JSON}; use ethereum_types::Address; @@ -180,14 +180,13 @@ pub async fn update_nft(ctx: MmArc, req: UpdateNftReq) -> MmResult<(), UpdateNft } update_nft_list(ctx.clone(), &storage, chain, scanned_block + 1, &req.url).await?; update_transfers_with_empty_meta(&storage, chain, &req.url).await?; - update_spam_phishing(&ctx, &storage, *chain, &req.url_antispam).await?; + update_spam(&ctx, &storage, *chain, &req.url_antispam).await?; } Ok(()) } -#[allow(dead_code)] -/// `update_spam_phishing` function updates spam contracts and phishing domains info in NFT list and NFT transfers. -async fn update_spam_phishing( +/// `update_spam_phishing` function updates spam contracts info in NFT list and NFT transfers. +async fn update_spam( ctx: &MmArc, storage: &T, chain: Chain, @@ -199,11 +198,34 @@ where if chain == Chain::Eth || chain == Chain::Polygon { update_spam_nft_with_mnemonichq(ctx, storage, &chain, url_antispam).await?; } - let _scan_contract_uri = prepare_uri_for_blocklist_endpoint(url_antispam, BLOCKLIST_CONTRACT, BLOCKLIST_SCAN)?; - // todo get all unique contracts from transfer table (as it also covers NFT list table) and send post req with these addresses. - // todo most likely `update_spam_phishing` will be renamed to `update_spam` as there will be too many logic and code with phishing impl. - todo!() + let scan_contract_uri = prepare_uri_for_blocklist_endpoint(url_antispam, BLOCKLIST_CONTRACT, BLOCKLIST_SCAN)?; + let token_addresses = storage.get_token_addresses(&chain).await?; + let addresses = token_addresses + .iter() + .map(eth_addr_to_hex) + .collect::>() + .join(","); + let req_spam = SpamContractReq { + network: chain, + addresses, + }; + let req_spam_json = serde_json::to_string(&req_spam)?; + let scan_contract_res = send_post_request_to_uri(scan_contract_uri.as_str(), req_spam_json).await?; + let spam_res: SpamContractRes = serde_json::from_slice(&scan_contract_res)?; + for (address, is_spam) in spam_res.result.into_iter() { + if is_spam { + let address_hex = eth_addr_to_hex(&address); + storage + .update_nft_spam_by_token_address(&chain, address_hex.clone(), is_spam) + .await?; + storage + .update_transfer_spam_by_token_address(&chain, address_hex, is_spam) + .await?; + } + } + Ok(()) } + /// Uses `/api/blocklist/wallet/{network}/{wallet_address}` endpoint to get **all spam contract addresses** of NFTs owned by the user. /// Currently, only the `Eth` and `Polygon` networks are supported. async fn update_spam_nft_with_mnemonichq( diff --git a/mm2src/coins/nft/nft_structs.rs b/mm2src/coins/nft/nft_structs.rs index 4b13ed48bc..925d799c78 100644 --- a/mm2src/coins/nft/nft_structs.rs +++ b/mm2src/coins/nft/nft_structs.rs @@ -538,7 +538,6 @@ pub(crate) struct PhishingDomainReq { pub(crate) domains: String, } -#[allow(dead_code)] #[derive(Debug, Deserialize)] pub(crate) struct SpamContractRes { pub(crate) result: HashMap, diff --git a/mm2src/coins/nft/nft_tests.rs b/mm2src/coins/nft/nft_tests.rs index d1351d50c1..3534d5cb3e 100644 --- a/mm2src/coins/nft/nft_tests.rs +++ b/mm2src/coins/nft/nft_tests.rs @@ -149,10 +149,10 @@ mod native_tests { fn test_refresh_metadata() { block_on(test_refresh_metadata_impl()) } #[test] - fn test_nft_spam_by_token_address() { block_on(test_update_nft_spam_by_token_address_impl()) } + fn test_nft_amount() { block_on(test_nft_amount_impl()) } #[test] - fn test_nft_amount() { block_on(test_nft_amount_impl()) } + fn test_nft_spam_by_token_address() { block_on(test_update_nft_spam_by_token_address_impl()) } #[test] fn test_add_get_transfers() { block_on(test_add_get_transfers_impl()) } @@ -171,6 +171,9 @@ mod native_tests { #[test] fn test_update_transfer_spam_by_token_address() { block_on(test_update_transfer_spam_by_token_address_impl()) } + + #[test] + fn test_get_token_addresses() { block_on(test_get_token_addresses_impl()) } } #[cfg(target_arch = "wasm32")] @@ -271,6 +274,9 @@ mod wasm_tests { #[wasm_bindgen_test] async fn test_refresh_metadata() { test_refresh_metadata_impl().await } + #[wasm_bindgen_test] + async fn test_nft_spam_by_token_address() { test_update_nft_spam_by_token_address_impl().await } + #[wasm_bindgen_test] async fn test_add_get_transfers() { test_add_get_transfers_impl().await } @@ -285,4 +291,10 @@ mod wasm_tests { #[wasm_bindgen_test] async fn test_get_update_transfer_meta() { test_get_update_transfer_meta_impl().await } + + #[wasm_bindgen_test] + async fn test_update_transfer_spam_by_token_address() { test_update_transfer_spam_by_token_address_impl().await } + + #[wasm_bindgen_test] + async fn test_get_token_addresses() { test_get_token_addresses_impl().await } } diff --git a/mm2src/coins/nft/storage/db_test_helpers.rs b/mm2src/coins/nft/storage/db_test_helpers.rs index a105e48392..f26409dd19 100644 --- a/mm2src/coins/nft/storage/db_test_helpers.rs +++ b/mm2src/coins/nft/storage/db_test_helpers.rs @@ -515,7 +515,6 @@ pub(crate) async fn test_refresh_metadata_impl() { assert_eq!(new_symbol.to_string(), nft_upd.common.symbol.unwrap()); } -#[allow(dead_code)] pub(crate) async fn test_update_nft_spam_by_token_address_impl() { let chain = Chain::Bsc; let storage = init_nft_list_storage(&chain).await; @@ -692,7 +691,6 @@ pub(crate) async fn test_get_update_transfer_meta_impl() { assert_eq!(transfer_by_hash.token_name, Some("Nebula Nodes".to_string())) } -#[allow(dead_code)] pub(crate) async fn test_update_transfer_spam_by_token_address_impl() { let chain = Chain::Bsc; let storage = init_nft_history_storage(&chain).await; @@ -711,3 +709,13 @@ pub(crate) async fn test_update_transfer_spam_by_token_address_impl() { assert!(transfers.common.possible_spam); } } + +pub(crate) async fn test_get_token_addresses_impl() { + let chain = Chain::Bsc; + let storage = init_nft_history_storage(&chain).await; + let transfers = nft_transfer_history(); + storage.add_transfers_to_history(&chain, transfers).await.unwrap(); + + let token_addresses = storage.get_token_addresses(&chain).await.unwrap(); + assert_eq!(token_addresses.len(), 2); +} diff --git a/mm2src/coins/nft/storage/mod.rs b/mm2src/coins/nft/storage/mod.rs index 6868a2ba46..7aaf7402c0 100644 --- a/mm2src/coins/nft/storage/mod.rs +++ b/mm2src/coins/nft/storage/mod.rs @@ -3,6 +3,7 @@ use crate::nft::nft_structs::{Chain, Nft, NftList, NftTokenAddrId, NftTransferHi use crate::WithdrawError; use async_trait::async_trait; use derive_more::Display; +use ethereum_types::Address; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::mm_error::MmResult; use mm2_err_handle::mm_error::{NotEqual, NotMmError}; @@ -174,6 +175,9 @@ pub trait NftTransferHistoryStorageOps { token_address: String, possible_spam: bool, ) -> MmResult<(), Self::Error>; + + /// `get_token_addresses` return all unique token addresses. + async fn get_token_addresses(&self, chain: &Chain) -> MmResult, Self::Error>; } #[derive(Debug, Deserialize, Display, Serialize)] diff --git a/mm2src/coins/nft/storage/sql_storage.rs b/mm2src/coins/nft/storage/sql_storage.rs index 0d40fdc029..7a6e988e3b 100644 --- a/mm2src/coins/nft/storage/sql_storage.rs +++ b/mm2src/coins/nft/storage/sql_storage.rs @@ -10,6 +10,7 @@ use db_common::sqlite::rusqlite::types::{FromSqlError, Type}; use db_common::sqlite::rusqlite::{Connection, Error as SqlError, Row, Statement}; use db_common::sqlite::sql_builder::SqlBuilder; use db_common::sqlite::{query_single_row, string_from_row, validate_table_name, CHECK_TABLE_EXISTS_SQL}; +use ethereum_types::Address; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::map_to_mm::MapToMmResult; use mm2_err_handle::mm_error::{MmError, MmResult}; @@ -212,6 +213,13 @@ fn transfer_history_from_row(row: &Row<'_>) -> Result) -> Result { + let address: String = row.get(0)?; + address + .parse() + .map_err(|e| SqlError::FromSqlConversionFailure(0, Type::Text, Box::new(e))) +} + fn token_address_id_from_row(row: &Row<'_>) -> Result { let token_address: String = row.get("token_address")?; let token_id_str: String = row.get("token_id")?; @@ -390,6 +398,21 @@ where Ok(stmt) } +fn get_token_addresses_statement<'a, F>( + conn: &'a Connection, + chain: &'a Chain, + table_name_creator: F, +) -> MmResult, SqlError> +where + F: FnOnce(&Chain) -> String, +{ + let table_name = table_name_creator(chain); + validate_table_name(table_name.as_str())?; + let sql_query = format!("SELECT DISTINCT token_address FROM {}", table_name); + let stmt = conn.prepare(&sql_query)?; + Ok(stmt) +} + fn get_transfers_from_block_builder<'a>( conn: &'a Connection, chain: &'a Chain, @@ -1048,4 +1071,16 @@ impl NftTransferHistoryStorageOps for SqliteNftStorage { }) .await } + + async fn get_token_addresses(&self, chain: &Chain) -> MmResult, Self::Error> { + let selfi = self.clone(); + let chain = *chain; + async_blocking(move || { + let conn = selfi.0.lock().unwrap(); + let mut stmt = get_token_addresses_statement(&conn, &chain, nft_transfer_history_table_name)?; + let addresses = stmt.query_map([], address_from_row)?.collect::, _>>()?; + Ok(addresses) + }) + .await + } } diff --git a/mm2src/coins/nft/storage/wasm/wasm_storage.rs b/mm2src/coins/nft/storage/wasm/wasm_storage.rs index 54e60a2772..bf679c0b2e 100644 --- a/mm2src/coins/nft/storage/wasm/wasm_storage.rs +++ b/mm2src/coins/nft/storage/wasm/wasm_storage.rs @@ -7,6 +7,7 @@ use crate::nft::storage::{get_offset_limit, CreateNftStorageError, NftListStorag NftTransferHistoryFilters, NftTransferHistoryStorageOps, RemoveNftResult}; use async_trait::async_trait; use common::is_initial_upgrade; +use ethereum_types::Address; use mm2_core::mm_ctx::MmArc; use mm2_db::indexed_db::{BeBigUint, DbTable, DbUpgrader, MultiIndex, OnUpgradeResult, SharedDb, TableSignature}; use mm2_err_handle::map_mm_error::MapMmError; @@ -316,21 +317,47 @@ impl NftListStorageOps for IndexedDbNftStorage { Ok(()) } - async fn get_nfts_by_token_address( - &self, - _chain: &Chain, - _token_address: String, - ) -> MmResult, Self::Error> { - todo!() + async fn get_nfts_by_token_address(&self, chain: &Chain, token_address: String) -> MmResult, Self::Error> { + let locked_db = self.lock_db().await?; + let db_transaction = locked_db.get_inner().transaction().await?; + let table = db_transaction.table::().await?; + + let index_keys = MultiIndex::new(NftListTable::CHAIN_TOKEN_ADD_INDEX) + .with_value(chain.to_string())? + .with_value(&token_address)?; + + table + .get_items_by_multi_index(index_keys) + .await? + .into_iter() + .map(|(_item_id, item)| nft_details_from_item(item)) + .collect() } async fn update_nft_spam_by_token_address( &self, - _chain: &Chain, - _token_address: String, - _possible_spam: bool, + chain: &Chain, + token_address: String, + possible_spam: bool, ) -> MmResult<(), Self::Error> { - todo!() + let nfts: Vec = self.get_nfts_by_token_address(chain, token_address.clone()).await?; + let locked_db = self.lock_db().await?; + let db_transaction = locked_db.get_inner().transaction().await?; + let table = db_transaction.table::().await?; + + for mut nft in nfts { + nft.common.possible_spam = possible_spam; + drop_mutability!(nft); + + let index_keys = MultiIndex::new(NftListTable::CHAIN_TOKEN_ADD_TOKEN_ID_INDEX) + .with_value(chain.to_string())? + .with_value(eth_addr_to_hex(&nft.common.token_address))? + .with_value(nft.common.token_id.to_string())?; + + let item = NftListTable::from_nft(&nft)?; + table.replace_item_by_unique_multi_index(index_keys, &item).await?; + } + Ok(()) } } @@ -541,19 +568,65 @@ impl NftTransferHistoryStorageOps for IndexedDbNftStorage { async fn get_transfers_by_token_address( &self, - _chain: &Chain, - _token_address: String, + chain: &Chain, + token_address: String, ) -> MmResult, Self::Error> { - todo!() + let locked_db = self.lock_db().await?; + let db_transaction = locked_db.get_inner().transaction().await?; + let table = db_transaction.table::().await?; + + let index_keys = MultiIndex::new(NftTransferHistoryTable::CHAIN_TOKEN_ADD_INDEX) + .with_value(chain.to_string())? + .with_value(&token_address)?; + + table + .get_items_by_multi_index(index_keys) + .await? + .into_iter() + .map(|(_item_id, item)| transfer_details_from_item(item)) + .collect() } async fn update_transfer_spam_by_token_address( &self, - _chain: &Chain, - _token_address: String, - _possible_spam: bool, + chain: &Chain, + token_address: String, + possible_spam: bool, ) -> MmResult<(), Self::Error> { - todo!() + let transfers: Vec = self + .get_transfers_by_token_address(chain, token_address.clone()) + .await?; + let locked_db = self.lock_db().await?; + let db_transaction = locked_db.get_inner().transaction().await?; + let table = db_transaction.table::().await?; + + for mut transfer in transfers { + transfer.common.possible_spam = possible_spam; + drop_mutability!(transfer); + + let index_keys = MultiIndex::new(NftTransferHistoryTable::CHAIN_TX_HASH_LOG_INDEX_INDEX) + .with_value(chain.to_string())? + .with_value(&transfer.common.transaction_hash)? + .with_value(transfer.common.log_index)?; + + let item = NftTransferHistoryTable::from_transfer_history(&transfer)?; + table.replace_item_by_unique_multi_index(index_keys, &item).await?; + } + Ok(()) + } + + async fn get_token_addresses(&self, chain: &Chain) -> MmResult, Self::Error> { + let locked_db = self.lock_db().await?; + let db_transaction = locked_db.get_inner().transaction().await?; + let table = db_transaction.table::().await?; + + let items = table.get_items("chain", chain.to_string()).await?; + let mut token_addresses = HashSet::new(); + for (_item_id, item) in items.into_iter() { + let transfer = transfer_details_from_item(item)?; + token_addresses.insert(transfer.common.token_address); + } + Ok(token_addresses.into_iter().collect()) } } @@ -611,6 +684,7 @@ pub(crate) struct NftListTable { amount: String, block_number: BeBigUint, contract_type: ContractType, + possible_spam: bool, details_json: Json, } @@ -619,6 +693,8 @@ impl NftListTable { const CHAIN_BLOCK_NUMBER_INDEX: &str = "chain_block_number_index"; + const CHAIN_TOKEN_ADD_INDEX: &str = "chain_token_add_index"; + fn from_nft(nft: &Nft) -> WasmNftCacheResult { let details_json = json::to_value(nft).map_to_mm(|e| WasmNftCacheError::ErrorSerializing(e.to_string()))?; Ok(NftListTable { @@ -628,6 +704,7 @@ impl NftListTable { amount: nft.common.amount.to_string(), block_number: BeBigUint::from(nft.block_number), contract_type: nft.contract_type, + possible_spam: nft.common.possible_spam, details_json, }) } @@ -645,6 +722,7 @@ impl TableSignature for NftListTable { true, )?; table.create_multi_index(Self::CHAIN_BLOCK_NUMBER_INDEX, &["chain", "block_number"], false)?; + table.create_multi_index(Self::CHAIN_TOKEN_ADD_INDEX, &["chain", "token_address"], false)?; table.create_index("chain", false)?; table.create_index("block_number", false)?; } @@ -668,6 +746,7 @@ pub(crate) struct NftTransferHistoryTable { collection_name: Option, image_url: Option, token_name: Option, + possible_spam: bool, details_json: Json, } @@ -678,6 +757,8 @@ impl NftTransferHistoryTable { const CHAIN_BLOCK_NUMBER_INDEX: &str = "chain_block_number_index"; + const CHAIN_TOKEN_ADD_INDEX: &str = "chain_token_add_index"; + fn from_transfer_history(transfer: &NftTransferHistory) -> WasmNftCacheResult { let details_json = json::to_value(transfer).map_to_mm(|e| WasmNftCacheError::ErrorSerializing(e.to_string()))?; @@ -696,6 +777,7 @@ impl NftTransferHistoryTable { collection_name: transfer.collection_name.clone(), image_url: transfer.image_url.clone(), token_name: transfer.token_name.clone(), + possible_spam: transfer.common.possible_spam, details_json, }) } @@ -718,6 +800,7 @@ impl TableSignature for NftTransferHistoryTable { true, )?; table.create_multi_index(Self::CHAIN_BLOCK_NUMBER_INDEX, &["chain", "block_number"], false)?; + table.create_multi_index(Self::CHAIN_TOKEN_ADD_INDEX, &["chain", "token_address"], false)?; table.create_index("block_number", false)?; table.create_index("chain", false)?; } From a1f8655237d023c6062aa4b815a469e8f99fca69 Mon Sep 17 00:00:00 2001 From: laruh Date: Fri, 1 Sep 2023 19:26:39 +0700 Subject: [PATCH 03/39] use separated fields in nft sql tables --- mm2src/coins/nft.rs | 22 + mm2src/coins/nft/nft_structs.rs | 17 +- mm2src/coins/nft/storage/db_test_helpers.rs | 73 ++- mm2src/coins/nft/storage/mod.rs | 31 +- mm2src/coins/nft/storage/sql_storage.rs | 450 ++++++++++++------ mm2src/coins/nft/storage/wasm/wasm_storage.rs | 21 +- 6 files changed, 392 insertions(+), 222 deletions(-) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index 539b76497e..c9b62c1f4a 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -181,6 +181,7 @@ pub async fn update_nft(ctx: MmArc, req: UpdateNftReq) -> MmResult<(), UpdateNft update_nft_list(ctx.clone(), &storage, chain, scanned_block + 1, &req.url).await?; update_transfers_with_empty_meta(&storage, chain, &req.url).await?; update_spam(&ctx, &storage, *chain, &req.url_antispam).await?; + // todo update phishing domains } Ok(()) } @@ -273,6 +274,7 @@ fn prepare_uri_for_blocklist_endpoint( Ok(uri) } +/// `refresh_nft_metadata` function refreshes metadata related to NFT with specified token address and token id. pub async fn refresh_nft_metadata(ctx: MmArc, req: RefreshMetadataReq) -> MmResult<(), UpdateNftError> { let nft_ctx = NftCtx::from_ctx(&ctx).map_to_mm(GetNftInfoError::Internal)?; let _lock = nft_ctx.guard.lock().await; @@ -293,14 +295,17 @@ pub async fn refresh_nft_metadata(ctx: MmArc, req: RefreshMetadataReq) -> MmResu }; let mut nft_db = get_nft_metadata(ctx.clone(), req).await?; let token_uri = check_moralis_ipfs_bafy(moralis_meta.common.token_uri.as_deref()); + let token_domain = get_domain_from_url(token_uri.as_deref()); let uri_meta = get_uri_meta(token_uri.as_deref(), moralis_meta.common.metadata.as_deref()).await; nft_db.common.collection_name = moralis_meta.common.collection_name; nft_db.common.symbol = moralis_meta.common.symbol; nft_db.common.token_uri = token_uri; + nft_db.common.token_domain = token_domain; nft_db.common.metadata = moralis_meta.common.metadata; nft_db.common.last_token_uri_sync = moralis_meta.common.last_token_uri_sync; nft_db.common.last_metadata_sync = moralis_meta.common.last_metadata_sync; nft_db.common.possible_spam = moralis_meta.common.possible_spam; + // todo also update possible_phishing and check spam with antispam proxy additionally nft_db.uri_meta = uri_meta; drop_mutability!(nft_db); storage @@ -427,8 +432,10 @@ async fn get_moralis_nft_transfers( block_timestamp, contract_type, token_uri: None, + token_domain: None, collection_name: None, image_url: None, + image_domain: None, token_name: None, status, }; @@ -590,7 +597,11 @@ async fn get_uri_meta(token_uri: Option<&str>, metadata: Option<&str>) -> UriMet } } uri_meta.image_url = check_moralis_ipfs_bafy(uri_meta.image_url.as_deref()); + uri_meta.image_domain = get_domain_from_url(uri_meta.image_url.as_deref()); uri_meta.animation_url = check_moralis_ipfs_bafy(uri_meta.animation_url.as_deref()); + uri_meta.animation_domain = get_domain_from_url(uri_meta.animation_url.as_deref()); + uri_meta.external_url = check_moralis_ipfs_bafy(uri_meta.external_url.as_deref()); + uri_meta.external_domain = get_domain_from_url(uri_meta.external_url.as_deref()); drop_mutability!(uri_meta); uri_meta } @@ -817,6 +828,7 @@ async fn handle_receive_erc1155 Nft { let token_uri = check_moralis_ipfs_bafy(nft_moralis.common.token_uri.as_deref()); let uri_meta = get_uri_meta(token_uri.as_deref(), nft_moralis.common.metadata.as_deref()).await; + let token_domain = get_domain_from_url(token_uri.as_deref()); Nft { common: NftCommon { token_address: nft_moralis.common.token_address, @@ -1030,6 +1044,7 @@ async fn build_nft_from_moralis(chain: &Chain, nft_moralis: NftFromMoralis, cont collection_name: nft_moralis.common.collection_name, symbol: nft_moralis.common.symbol, token_uri, + token_domain, metadata: nft_moralis.common.metadata, last_token_uri_sync: nft_moralis.common.last_token_uri_sync, last_metadata_sync: nft_moralis.common.last_metadata_sync, @@ -1044,3 +1059,10 @@ async fn build_nft_from_moralis(chain: &Chain, nft_moralis: NftFromMoralis, cont uri_meta, } } + +#[inline(always)] +fn get_domain_from_url(url: Option<&str>) -> Option { + url.as_ref() + .and_then(|uri| Url::parse(uri).ok()) + .and_then(|url| url.domain().map(String::from)) +} diff --git a/mm2src/coins/nft/nft_structs.rs b/mm2src/coins/nft/nft_structs.rs index 925d799c78..2d4d467dff 100644 --- a/mm2src/coins/nft/nft_structs.rs +++ b/mm2src/coins/nft/nft_structs.rs @@ -185,12 +185,15 @@ pub(crate) struct UriMeta { #[serde(rename = "image")] pub(crate) raw_image_url: Option, pub(crate) image_url: Option, + pub(crate) image_domain: Option, #[serde(rename = "name")] pub(crate) token_name: Option, pub(crate) description: Option, pub(crate) attributes: Option, pub(crate) animation_url: Option, + pub(crate) animation_domain: Option, pub(crate) external_url: Option, + pub(crate) external_domain: Option, pub(crate) image_details: Option, } @@ -224,7 +227,6 @@ impl UriMeta { } } -#[allow(dead_code)] /// [`NftCommon`] structure contains common fields from [`Nft`] and [`NftFromMoralis`] #[derive(Clone, Debug, Deserialize, Serialize)] pub struct NftCommon { @@ -237,6 +239,7 @@ pub struct NftCommon { pub(crate) collection_name: Option, pub(crate) symbol: Option, pub(crate) token_uri: Option, + pub(crate) token_domain: Option, pub(crate) metadata: Option, pub(crate) last_token_uri_sync: Option, pub(crate) last_metadata_sync: Option, @@ -401,14 +404,13 @@ impl fmt::Display for TransferStatus { } } -#[allow(dead_code)] /// [`NftTransferCommon`] structure contains common fields from [`NftTransferHistory`] and [`NftTransferHistoryFromMoralis`] #[derive(Clone, Debug, Deserialize, Serialize)] pub struct NftTransferCommon { pub(crate) block_hash: Option, /// Transaction hash in hexadecimal format pub(crate) transaction_hash: String, - pub(crate) transaction_index: Option, + pub(crate) transaction_index: Option, pub(crate) log_index: u32, pub(crate) value: Option, pub(crate) transaction_type: Option, @@ -417,7 +419,7 @@ pub struct NftTransferCommon { pub(crate) from_address: Address, pub(crate) to_address: Address, pub(crate) amount: BigDecimal, - pub(crate) verified: Option, + pub(crate) verified: Option, pub(crate) operator: Option, #[serde(default)] pub(crate) possible_spam: bool, @@ -434,8 +436,10 @@ pub struct NftTransferHistory { pub(crate) block_timestamp: u64, pub(crate) contract_type: ContractType, pub(crate) token_uri: Option, + pub(crate) token_domain: Option, pub(crate) collection_name: Option, pub(crate) image_url: Option, + pub(crate) image_domain: Option, pub(crate) token_name: Option, pub(crate) status: TransferStatus, } @@ -472,7 +476,6 @@ pub struct NftTransferHistoryFilters { pub(crate) exclude_phishing: bool, } -#[allow(dead_code)] #[derive(Debug, Deserialize)] pub struct UpdateNftReq { pub(crate) chains: Vec, @@ -491,8 +494,10 @@ pub struct TransferMeta { pub(crate) token_address: String, pub(crate) token_id: BigDecimal, pub(crate) token_uri: Option, + pub(crate) token_domain: Option, pub(crate) collection_name: Option, pub(crate) image_url: Option, + pub(crate) image_domain: Option, pub(crate) token_name: Option, } @@ -502,8 +507,10 @@ impl From for TransferMeta { token_address: eth_addr_to_hex(&nft_db.common.token_address), token_id: nft_db.common.token_id, token_uri: nft_db.common.token_uri, + token_domain: nft_db.common.token_domain, collection_name: nft_db.common.collection_name, image_url: nft_db.uri_meta.image_url, + image_domain: nft_db.uri_meta.image_domain, token_name: nft_db.uri_meta.token_name, } } diff --git a/mm2src/coins/nft/storage/db_test_helpers.rs b/mm2src/coins/nft/storage/db_test_helpers.rs index f26409dd19..7b7ce13f64 100644 --- a/mm2src/coins/nft/storage/db_test_helpers.rs +++ b/mm2src/coins/nft/storage/db_test_helpers.rs @@ -30,6 +30,7 @@ pub(crate) fn nft() -> Nft { collection_name: None, symbol: None, token_uri: Some("https://tikimetadata.s3.amazonaws.com/tiki_box.json".to_string()), + token_domain: None, metadata: Some( "{\"name\":\"https://arweave.net\",\"image\":\"https://tikimetadata.s3.amazonaws.com/tiki_box.png\"}" .to_string(), @@ -52,43 +53,15 @@ pub(crate) fn nft() -> Nft { description: Some("Born to usher in Bull markets.".to_string()), attributes: None, animation_url: None, + animation_domain: None, external_url: None, + external_domain: None, image_details: None, + image_domain: None, }, } } -fn transfer() -> NftTransferHistory { - NftTransferHistory { - common: NftTransferCommon { - block_hash: Some("0x3d68b78391fb3cf8570df27036214f7e9a5a6a45d309197936f51d826041bfe7".to_string()), - transaction_hash: "0x1e9f04e9b571b283bde02c98c2a97da39b2bb665b57c1f2b0b733f9b681debbe".to_string(), - transaction_index: Some(198), - log_index: 495, - value: Default::default(), - transaction_type: Some("Single".to_string()), - token_address: Address::from_str("0xfd913a305d70a60aac4faac70c739563738e1f81").unwrap(), - token_id: BigDecimal::from_str("214300047252").unwrap(), - from_address: Address::from_str("0x6fad0ec6bb76914b2a2a800686acc22970645820").unwrap(), - to_address: Address::from_str("0xf622a6c52c94b500542e2ae6bcad24c53bc5b6a2").unwrap(), - amount: BigDecimal::from_str("1").unwrap(), - verified: Some(1), - operator: None, - possible_spam: false, - possible_phishing: false, - }, - chain: Chain::Bsc, - block_number: 28056726, - block_timestamp: 1683627432, - contract_type: ContractType::Erc721, - token_uri: None, - collection_name: Some("Binance NFT Mystery Box-Back to Blockchain Future".to_string()), - image_url: Some("https://public.nftstatic.com/static/nft/res/4df0a5da04174e1e9be04b22a805f605.png".to_string()), - token_name: Some("Nebula Nodes".to_string()), - status: TransferStatus::Receive, - } -} - fn nft_list() -> Vec { let nft = Nft { common: NftCommon { @@ -100,6 +73,7 @@ fn nft_list() -> Vec { collection_name: None, symbol: None, token_uri: Some("https://tikimetadata.s3.amazonaws.com/tiki_box.json".to_string()), + token_domain: None, metadata: Some("{\"name\":\"Tiki box\"}".to_string()), last_token_uri_sync: Some("2023-02-07T17:10:08.402Z".to_string()), last_metadata_sync: Some("2023-02-07T17:10:16.858Z".to_string()), @@ -118,8 +92,11 @@ fn nft_list() -> Vec { description: Some("Born to usher in Bull markets.".to_string()), attributes: None, animation_url: None, + animation_domain: None, external_url: None, + external_domain: None, image_details: None, + image_domain: None, }, }; @@ -133,6 +110,7 @@ fn nft_list() -> Vec { collection_name: Some("Binance NFT Mystery Box-Back to Blockchain Future".to_string()), symbol: Some("BMBBBF".to_string()), token_uri: Some("https://public.nftstatic.com/static/nft/BSC/BMBBBF/214300047252".to_string()), + token_domain: None, metadata: Some( "{\"image\":\"https://public.nftstatic.com/static/nft/res/4df0a5da04174e1e9be04b22a805f605.png\"}" .to_string(), @@ -157,8 +135,11 @@ fn nft_list() -> Vec { description: Some("Interchain nodes".to_string()), attributes: None, animation_url: None, + animation_domain: None, external_url: None, + external_domain: None, image_details: None, + image_domain: None, }, }; @@ -172,6 +153,7 @@ fn nft_list() -> Vec { collection_name: Some("Binance NFT Mystery Box-Back to Blockchain Future".to_string()), symbol: Some("BMBBBF".to_string()), token_uri: Some("https://public.nftstatic.com/static/nft/BSC/BMBBBF/214300047252".to_string()), + token_domain: None, metadata: Some( "{\"image\":\"https://public.nftstatic.com/static/nft/res/4df0a5da04174e1e9be04b22a805f605.png\"}" .to_string(), @@ -196,8 +178,11 @@ fn nft_list() -> Vec { description: Some("Interchain nodes".to_string()), attributes: None, animation_url: None, + animation_domain: None, external_url: None, + external_domain: None, image_details: None, + image_domain: None, }, }; @@ -211,6 +196,7 @@ fn nft_list() -> Vec { collection_name: Some("Binance NFT Mystery Box-Back to Blockchain Future".to_string()), symbol: Some("BMBBBF".to_string()), token_uri: Some("https://public.nftstatic.com/static/nft/BSC/BMBBBF/214300044414".to_string()), + token_domain: None, metadata: Some( "{\"image\":\"https://public.nftstatic.com/static/nft/res/4df0a5da04174e1e9be04b22a805f605.png\"}" .to_string(), @@ -235,8 +221,11 @@ fn nft_list() -> Vec { description: Some("Interchain nodes".to_string()), attributes: None, animation_url: None, + animation_domain: None, external_url: None, + external_domain: None, image_details: None, + image_domain: None, }, }; vec![nft, nft1, nft2, nft3] @@ -266,8 +255,10 @@ fn nft_transfer_history() -> Vec { block_timestamp: 1677166110, contract_type: ContractType::Erc1155, token_uri: None, + token_domain: None, collection_name: None, image_url: None, + image_domain: None, token_name: None, status: TransferStatus::Receive, }; @@ -296,8 +287,10 @@ fn nft_transfer_history() -> Vec { contract_type: ContractType::Erc721, token_uri: None, + token_domain: None, collection_name: None, image_url: None, + image_domain: None, token_name: None, status: TransferStatus::Receive, @@ -328,8 +321,10 @@ fn nft_transfer_history() -> Vec { contract_type: ContractType::Erc721, token_uri: None, + token_domain: None, collection_name: None, image_url: None, + image_domain: None, token_name: None, status: TransferStatus::Receive, @@ -360,8 +355,10 @@ fn nft_transfer_history() -> Vec { contract_type: ContractType::Erc721, token_uri: None, + token_domain: None, collection_name: Some("Binance NFT Mystery Box-Back to Blockchain Future".to_string()), image_url: Some("https://public.nftstatic.com/static/nft/res/4df0a5da04174e1e9be04b22a805f605.png".to_string()), + image_domain: None, token_name: Some("Nebula Nodes".to_string()), status: TransferStatus::Receive, @@ -663,8 +660,10 @@ pub(crate) async fn test_get_update_transfer_meta_impl() { token_address: token_add.clone(), token_id: Default::default(), token_uri: None, + token_domain: None, collection_name: None, image_url: None, + image_domain: None, token_name: Some("Tiki box".to_string()), }; storage @@ -677,18 +676,6 @@ pub(crate) async fn test_get_update_transfer_meta_impl() { .unwrap(); let transfer_upd = transfer_upd.get(0).unwrap(); assert_eq!(transfer_upd.token_name, Some("Tiki box".to_string())); - - let transfer_meta = transfer(); - storage - .update_transfer_meta_by_hash_and_log_index(&chain, transfer_meta) - .await - .unwrap(); - let transfer_by_hash = storage - .get_transfer_by_tx_hash_and_log_index(&chain, TX_HASH.to_string(), LOG_INDEX) - .await - .unwrap() - .unwrap(); - assert_eq!(transfer_by_hash.token_name, Some("Nebula Nodes".to_string())) } pub(crate) async fn test_update_transfer_spam_by_token_address_impl() { diff --git a/mm2src/coins/nft/storage/mod.rs b/mm2src/coins/nft/storage/mod.rs index 7aaf7402c0..64a9460c92 100644 --- a/mm2src/coins/nft/storage/mod.rs +++ b/mm2src/coins/nft/storage/mod.rs @@ -147,12 +147,6 @@ pub trait NftTransferHistoryStorageOps { log_index: u32, ) -> MmResult, Self::Error>; - async fn update_transfer_meta_by_hash_and_log_index( - &self, - chain: &Chain, - transfer: NftTransferHistory, - ) -> MmResult<(), Self::Error>; - async fn update_transfers_meta_by_token_addr_id( &self, chain: &Chain, @@ -223,3 +217,28 @@ fn get_offset_limit(max: bool, limit: usize, page_number: Option, None => (0, limit), } } + +/// `NftDetailsJson` structure contains immutable parameters that are not needed for queries. +/// This is what `details_json` string contains in db table. +#[derive(Clone, Debug, Default, Deserialize, Serialize)] +pub(crate) struct NftDetailsJson { + pub(crate) owner_of: Address, + pub(crate) token_hash: Option, + pub(crate) minter_address: Option, + pub(crate) block_number_minted: Option, +} + +#[allow(dead_code)] +/// `TransferDetailsJson` structure contains immutable parameters that are not needed for queries. +/// /// This is what `details_json` string contains in db table. +#[derive(Clone, Debug, Default, Deserialize, Serialize)] +pub(crate) struct TransferDetailsJson { + pub(crate) block_hash: Option, + pub(crate) transaction_index: Option, + pub(crate) value: Option, + pub(crate) transaction_type: Option, + pub(crate) verified: Option, + pub(crate) operator: Option, + pub(crate) from_address: Address, + pub(crate) to_address: Address, +} diff --git a/mm2src/coins/nft/storage/sql_storage.rs b/mm2src/coins/nft/storage/sql_storage.rs index 7a6e988e3b..d01d09ecfd 100644 --- a/mm2src/coins/nft/storage/sql_storage.rs +++ b/mm2src/coins/nft/storage/sql_storage.rs @@ -1,8 +1,9 @@ use crate::nft::eth_addr_to_hex; -use crate::nft::nft_structs::{Chain, ConvertChain, Nft, NftList, NftTokenAddrId, NftTransferHistory, - NftTransferHistoryFilters, NftsTransferHistoryList, TransferMeta}; -use crate::nft::storage::{get_offset_limit, CreateNftStorageError, NftListStorageOps, NftStorageError, - NftTransferHistoryStorageOps, RemoveNftResult}; +use crate::nft::nft_structs::{Chain, ContractType, ConvertChain, Nft, NftCommon, NftList, NftTokenAddrId, + NftTransferCommon, NftTransferHistory, NftTransferHistoryFilters, + NftsTransferHistoryList, TransferMeta, UriMeta}; +use crate::nft::storage::{get_offset_limit, CreateNftStorageError, NftDetailsJson, NftListStorageOps, NftStorageError, + NftTransferHistoryStorageOps, RemoveNftResult, TransferDetailsJson}; use async_trait::async_trait; use common::async_blocking; use db_common::sql_build::{SqlCondition, SqlQuery}; @@ -15,6 +16,7 @@ use mm2_core::mm_ctx::MmArc; use mm2_err_handle::map_to_mm::MapToMmResult; use mm2_err_handle::mm_error::{MmError, MmResult}; use mm2_number::BigDecimal; +use serde_json::Value as Json; use serde_json::{self as json}; use std::convert::TryInto; use std::num::NonZeroUsize; @@ -40,6 +42,24 @@ fn create_nft_list_table_sql(chain: &Chain) -> MmResult { contract_type TEXT NOT NULL, possible_spam INTEGER DEFAULT 0 NOT NULL, possible_phishing INTEGER DEFAULT 0 NOT NULL, + collection_name TEXT, + symbol TEXT, + token_uri TEXT, + token_domain TEXT, + metadata TEXT, + last_token_uri_sync TEXT, + last_metadata_sync TEXT, + raw_image_url TEXT, + image_url TEXT, + image_domain TEXT, + token_name TEXT, + description TEXT, + attributes TEXT, + animation_url TEXT, + animation_domain TEXT, + external_url TEXT, + external_domain TEXT, + image_details TEXT, details_json TEXT, PRIMARY KEY (token_address, token_id) );", @@ -63,12 +83,14 @@ fn create_transfer_history_table_sql(chain: &Chain) -> MmResult MmResult { +fn finalize_sql_builder(mut sql_builder: SqlBuilder, offset: usize, limit: usize) -> MmResult { let sql = sql_builder - .field("nft_list.details_json") + .field("*") .offset(offset) .limit(limit) .sql() @@ -189,28 +207,151 @@ fn finalize_nft_list_sql_builder( Ok(sql) } -fn finalize_nft_history_sql_builder( - mut sql_builder: SqlBuilder, - offset: usize, - limit: usize, -) -> MmResult { - let sql = sql_builder - .field("nft_history.details_json") - .offset(offset) - .limit(limit) - .sql() - .map_err(|e| SqlError::ToSqlConversionFailure(e.into()))?; - Ok(sql) +fn get_and_parse(row: &Row<'_>, column: &str) -> Result { + let value_str: String = row.get(column)?; + value_str.parse().map_err(|_| SqlError::from(FromSqlError::InvalidType)) } fn nft_from_row(row: &Row<'_>) -> Result { - let json_string: String = row.get(0)?; - json::from_str(&json_string).map_err(|e| SqlError::FromSqlConversionFailure(0, Type::Text, Box::new(e))) + let token_address = get_and_parse(row, "token_address")?; + let token_id = get_and_parse(row, "token_id")?; + let chain = get_and_parse(row, "chain")?; + let amount = get_and_parse(row, "amount")?; + let block_number: u64 = row.get("block_number")?; + let contract_type = get_and_parse(row, "contract_type")?; + let possible_spam: i32 = row.get("possible_spam")?; + let possible_phishing: i32 = row.get("possible_phishing")?; + let collection_name: Option = row.get("collection_name")?; + let symbol: Option = row.get("symbol")?; + let token_uri: Option = row.get("token_uri")?; + let token_domain: Option = row.get("token_domain")?; + let metadata: Option = row.get("metadata")?; + let last_token_uri_sync: Option = row.get("last_token_uri_sync")?; + let last_metadata_sync: Option = row.get("last_metadata_sync")?; + let raw_image_url: Option = row.get("raw_image_url")?; + let image_url: Option = row.get("image_url")?; + let image_domain: Option = row.get("image_domain")?; + let token_name: Option = row.get("token_name")?; + let description: Option = row.get("description")?; + let attributes_str: Option = row.get("attributes")?; + let attributes: Option = attributes_str + .as_deref() + .map(json::from_str) + .transpose() + .map_err(|e| SqlError::FromSqlConversionFailure(21, Type::Text, Box::new(e)))?; + let animation_url: Option = row.get("animation_url")?; + let animation_domain: Option = row.get("animation_domain")?; + let external_url: Option = row.get("external_url")?; + let external_domain: Option = row.get("external_domain")?; + let image_details_str: Option = row.get("image_details")?; + let image_details: Option = image_details_str + .as_deref() + .map(json::from_str) + .transpose() + .map_err(|e| SqlError::FromSqlConversionFailure(26, Type::Text, Box::new(e)))?; + let details_json: String = row.get("details_json")?; + let nft_details: NftDetailsJson = + json::from_str(&details_json).map_err(|e| SqlError::FromSqlConversionFailure(27, Type::Text, Box::new(e)))?; + + let uri_meta = UriMeta { + raw_image_url, + image_url, + image_domain, + token_name, + description, + attributes, + animation_url, + animation_domain, + external_url, + external_domain, + image_details, + }; + + let common = NftCommon { + token_address, + token_id, + amount, + owner_of: nft_details.owner_of, + token_hash: nft_details.token_hash, + collection_name, + symbol, + token_uri, + token_domain, + metadata, + last_token_uri_sync, + last_metadata_sync, + minter_address: nft_details.minter_address, + possible_spam: possible_spam != 0, + possible_phishing: possible_phishing != 0, + }; + let nft = Nft { + common, + chain, + block_number_minted: nft_details.block_number_minted, + block_number, + contract_type, + uri_meta, + }; + Ok(nft) } fn transfer_history_from_row(row: &Row<'_>) -> Result { - let json_string: String = row.get(0)?; - json::from_str(&json_string).map_err(|e| SqlError::FromSqlConversionFailure(0, Type::Text, Box::new(e))) + let transaction_hash: String = row.get("transaction_hash")?; + let log_index: u32 = row.get("log_index")?; + let chain: Chain = get_and_parse(row, "chain")?; + let block_number: u64 = row.get("block_number")?; + let block_timestamp: u64 = row.get("block_timestamp")?; + let contract_type: ContractType = get_and_parse(row, "contract_type")?; + let token_address: Address = get_and_parse(row, "token_address")?; + let token_id: BigDecimal = get_and_parse(row, "token_id")?; + let status = get_and_parse(row, "status")?; + let amount: BigDecimal = get_and_parse(row, "amount")?; + let token_uri: Option = row.get("token_uri")?; + let token_domain: Option = row.get("token_domain")?; + let collection_name: Option = row.get("collection_name")?; + let image_url: Option = row.get("image_url")?; + let image_domain: Option = row.get("image_domain")?; + let token_name: Option = row.get("token_name")?; + let possible_spam: i32 = row.get("possible_spam")?; + let possible_phishing: i32 = row.get("possible_phishing")?; + let details_json: String = row.get("details_json")?; + let details: TransferDetailsJson = + json::from_str(&details_json).map_err(|e| SqlError::FromSqlConversionFailure(19, Type::Text, Box::new(e)))?; + + let common = NftTransferCommon { + block_hash: details.block_hash, + transaction_hash, + transaction_index: details.transaction_index, + log_index, + value: details.value, + transaction_type: details.transaction_type, + token_address, + token_id, + from_address: details.from_address, + to_address: details.to_address, + amount, + verified: details.verified, + operator: details.operator, + possible_spam: possible_spam != 0, + possible_phishing: possible_phishing != 0, + }; + + let transfer_history = NftTransferHistory { + common, + chain, + block_number, + block_timestamp, + contract_type, + token_uri, + token_domain, + collection_name, + image_url, + image_domain, + token_name, + status, + }; + + Ok(transfer_history) } fn address_from_row(row: &Row<'_>) -> Result { @@ -237,9 +378,13 @@ fn insert_nft_in_list_sql(chain: &Chain) -> MmResult { let sql = format!( "INSERT INTO {} ( token_address, token_id, chain, amount, block_number, contract_type, possible_spam, - possible_phishing, details_json + possible_phishing, collection_name, symbol, token_uri, token_domain, metadata, + last_token_uri_sync, last_metadata_sync, raw_image_url, image_url, image_domain, + token_name, description, attributes, animation_url, animation_domain, external_url, + external_domain, image_details, details_json ) VALUES ( - ?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9 + ?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17, + ?18, ?19, ?20, ?21, ?22, ?23, ?24, ?25, ?26, ?27 );", table_name ); @@ -253,10 +398,10 @@ fn insert_transfer_in_history_sql(chain: &Chain) -> MmResult { let sql = format!( "INSERT INTO {} ( transaction_hash, log_index, chain, block_number, block_timestamp, contract_type, - token_address, token_id, status, amount, token_uri, collection_name, image_url, token_name, - possible_spam, possible_phishing, details_json + token_address, token_id, status, amount, token_uri, token_domain, collection_name, image_url, image_domain, + token_name, possible_spam, possible_phishing, details_json ) VALUES ( - ?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17 + ?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17, ?18, ?19 );", table_name ); @@ -273,26 +418,26 @@ fn upsert_last_scanned_block_sql() -> MmResult { Ok(sql) } -fn update_details_json_by_token_add_id_sql(chain: &Chain, table_name_creator: F) -> MmResult -where - F: FnOnce(&Chain) -> String, -{ - let table_name = table_name_creator(chain); +fn refresh_nft_metadata_sql(chain: &Chain) -> MmResult { + let table_name = nft_list_table_name(chain); validate_table_name(&table_name)?; let sql = format!( - "UPDATE {} SET details_json = ?1 WHERE token_address = ?2 AND token_id = ?3;", + "UPDATE {} SET possible_spam = ?1, possible_phishing = ?2, collection_name = ?3, symbol = ?4, token_uri = ?5, token_domain = ?6, metadata = ?7, \ + last_token_uri_sync = ?8, last_metadata_sync = ?9, raw_image_url = ?10, image_url = ?11, image_domain = ?12, token_name = ?13, description = ?14, \ + attributes = ?15, animation_url = ?16, animation_domain = ?17, external_url = ?18, external_domain = ?19, image_details = ?20 WHERE token_address = ?21 AND token_id = ?22;", table_name ); Ok(sql) } -fn update_meta_by_tx_hash_and_log_index_sql(chain: &Chain) -> MmResult { +fn update_transfers_meta_by_token_addr_id_sql(chain: &Chain) -> MmResult { let table_name = nft_transfer_history_table_name(chain); validate_table_name(&table_name)?; let sql = format!( - "UPDATE {} SET token_uri = ?1, collection_name = ?2, image_url = ?3, token_name = ?4, details_json = ?5 WHERE transaction_hash = ?6 AND log_index = ?7;", + "UPDATE {} SET token_uri = ?1, token_domain = ?2, collection_name = ?3, image_url = ?4, image_domain = ?5, \ + token_name = ?6 WHERE token_address = ?7 AND token_id = ?8;", table_name ); Ok(sql) @@ -306,7 +451,7 @@ where validate_table_name(&table_name)?; let sql = format!( - "UPDATE {} SET amount = ?1, details_json = ?2 WHERE token_address = ?3 AND token_id = ?4;", + "UPDATE {} SET amount = ?1 WHERE token_address = ?2 AND token_id = ?3;", table_name ); Ok(sql) @@ -320,7 +465,7 @@ where validate_table_name(&table_name)?; let sql = format!( - "UPDATE {} SET amount = ?1, block_number = ?2, details_json = ?3 WHERE token_address = ?4 AND token_id = ?5;", + "UPDATE {} SET amount = ?1, block_number = ?2 WHERE token_address = ?3 AND token_id = ?4;", table_name ); Ok(sql) @@ -329,10 +474,7 @@ where fn get_nft_metadata_sql(chain: &Chain) -> MmResult { let table_name = nft_list_table_name(chain); validate_table_name(&table_name)?; - let sql = format!( - "SELECT details_json FROM {} WHERE token_address=?1 AND token_id=?2", - table_name - ); + let sql = format!("SELECT * FROM {} WHERE token_address=?1 AND token_id=?2", table_name); Ok(sql) } @@ -393,7 +535,7 @@ where { let table_name = table_name_creator(chain); validate_table_name(table_name.as_str())?; - let sql_query = format!("SELECT details_json FROM {} WHERE token_address = ?", table_name); + let sql_query = format!("SELECT * FROM {} WHERE token_address = ?", table_name); let stmt = conn.prepare(&sql_query)?; Ok(stmt) } @@ -413,21 +555,32 @@ where Ok(stmt) } -fn get_transfers_from_block_builder<'a>( - conn: &'a Connection, - chain: &'a Chain, - from_block: u64, -) -> MmResult, SqlError> { +// fn get_transfers_from_block_builder<'a>( +// conn: &'a Connection, +// chain: &'a Chain, +// from_block: u64, +// ) -> MmResult, SqlError> { +// let table_name = nft_transfer_history_table_name(chain); +// validate_table_name(table_name.as_str())?; +// let mut sql_builder = SqlQuery::select_from(conn, table_name.as_str())?; +// sql_builder +// .sql_builder() +// .and_where(format!("block_number >= '{}'", from_block)) +// .order_asc("block_number") +// .field("*"); +// drop_mutability!(sql_builder); +// Ok(sql_builder) +// } + +fn get_transfers_from_block_statement<'a>(conn: &'a Connection, chain: &'a Chain) -> MmResult, SqlError> { let table_name = nft_transfer_history_table_name(chain); validate_table_name(table_name.as_str())?; - let mut sql_builder = SqlQuery::select_from(conn, table_name.as_str())?; - sql_builder - .sql_builder() - .and_where(format!("block_number >= '{}'", from_block)) - .order_asc("block_number") - .field("details_json"); - drop_mutability!(sql_builder); - Ok(sql_builder) + let sql_query = format!( + "SELECT * FROM {} WHERE block_number >= ? ORDER BY block_number ASC", + table_name + ); + let stmt = conn.prepare(&sql_query)?; + Ok(stmt) } fn get_transfers_by_token_addr_id_statement<'a>( @@ -436,10 +589,7 @@ fn get_transfers_by_token_addr_id_statement<'a>( ) -> MmResult, SqlError> { let table_name = nft_transfer_history_table_name(chain); validate_table_name(table_name.as_str())?; - let sql_query = format!( - "SELECT details_json FROM {} WHERE token_address = ? AND token_id = ?", - table_name - ); + let sql_query = format!("SELECT * FROM {} WHERE token_address = ? AND token_id = ?", table_name); let stmt = conn.prepare(&sql_query)?; Ok(stmt) } @@ -468,7 +618,7 @@ fn get_transfer_by_tx_hash_and_log_index_sql(chain: &Chain) -> MmResult MmResult<(), Self::Error> { - let sql = update_details_json_by_token_add_id_sql(chain, nft_list_table_name)?; - let nft_json = json::to_string(&nft).expect("serialization should not fail"); + let sql = refresh_nft_metadata_sql(chain)?; let selfi = self.clone(); async_blocking(move || { let mut conn = selfi.0.lock().unwrap(); let sql_transaction = conn.transaction()?; let params = [ - nft_json, - eth_addr_to_hex(&nft.common.token_address), - nft.common.token_id.to_string(), + Some(i32::from(nft.common.possible_spam).to_string()), + Some(i32::from(nft.common.possible_phishing).to_string()), + nft.common.collection_name, + nft.common.symbol, + nft.common.token_uri, + nft.common.token_domain, + nft.common.metadata, + nft.common.last_token_uri_sync, + nft.common.last_metadata_sync, + nft.uri_meta.raw_image_url, + nft.uri_meta.image_url, + nft.uri_meta.image_domain, + nft.uri_meta.token_name, + nft.uri_meta.description, + nft.uri_meta.attributes.map(|v| v.to_string()), + nft.uri_meta.animation_url, + nft.uri_meta.animation_domain, + nft.uri_meta.external_url, + nft.uri_meta.external_domain, + nft.uri_meta.image_details.map(|v| v.to_string()), + Some(eth_addr_to_hex(&nft.common.token_address)), + Some(nft.common.token_id.to_string()), ]; sql_transaction.execute(&sql, params)?; sql_transaction.commit()?; @@ -687,7 +879,6 @@ impl NftListStorageOps for SqliteNftStorage { async fn update_nft_amount(&self, chain: &Chain, nft: Nft, scanned_block: u64) -> MmResult<(), Self::Error> { let sql = update_nft_amount_sql(chain, nft_list_table_name)?; - let nft_json = json::to_string(&nft).expect("serialization should not fail"); let scanned_block_params = [chain.to_ticker(), scanned_block.to_string()]; let selfi = self.clone(); async_blocking(move || { @@ -695,7 +886,6 @@ impl NftListStorageOps for SqliteNftStorage { let sql_transaction = conn.transaction()?; let params = [ Some(nft.common.amount.to_string()), - Some(nft_json), Some(eth_addr_to_hex(&nft.common.token_address)), Some(nft.common.token_id.to_string()), ]; @@ -709,7 +899,6 @@ impl NftListStorageOps for SqliteNftStorage { async fn update_nft_amount_and_block_number(&self, chain: &Chain, nft: Nft) -> MmResult<(), Self::Error> { let sql = update_nft_amount_and_block_number_sql(chain, nft_list_table_name)?; - let nft_json = json::to_string(&nft).expect("serialization should not fail"); let scanned_block_params = [chain.to_ticker(), nft.block_number.to_string()]; let selfi = self.clone(); async_blocking(move || { @@ -718,7 +907,6 @@ impl NftListStorageOps for SqliteNftStorage { let params = [ Some(nft.common.amount.to_string()), Some(nft.block_number.to_string()), - Some(nft_json), Some(eth_addr_to_hex(&nft.common.token_address)), Some(nft.common.token_id.to_string()), ]; @@ -751,30 +939,16 @@ impl NftListStorageOps for SqliteNftStorage { possible_spam: bool, ) -> MmResult<(), Self::Error> { let selfi = self.clone(); - let nfts = selfi.get_nfts_by_token_address(chain, token_address.clone()).await?; let table_name = nft_list_table_name(chain); validate_table_name(&table_name)?; - let sql = format!( - "UPDATE {} SET possible_spam = ?1, details_json = ?2 WHERE token_address = ?3 AND token_id = ?4;", - table_name - ); + let sql = format!("UPDATE {} SET possible_spam = ?1 WHERE token_address = ?2;", table_name); async_blocking(move || { let mut conn = selfi.0.lock().unwrap(); let sql_transaction = conn.transaction()?; - - for mut nft in nfts.into_iter() { - nft.common.possible_spam = possible_spam; - let nft_json = json::to_string(&nft).expect("serialization should not fail"); - let params = [ - Some(i32::from(possible_spam).to_string()), - Some(nft_json), - Some(token_address.to_string()), - Some(nft.common.token_id.to_string()), - ]; - sql_transaction.execute(&sql, params)?; - } + let params = [Some(i32::from(possible_spam).to_string()), Some(token_address.clone())]; + sql_transaction.execute(&sql, params)?; sql_transaction.commit()?; Ok(()) }) @@ -832,7 +1006,7 @@ impl NftTransferHistoryStorageOps for SqliteNftStorage { let count_total = total.try_into().expect("count should not be failed"); let (offset, limit) = get_offset_limit(max, limit, page_number, count_total); - let sql = finalize_nft_history_sql_builder(sql_builder, offset, limit)?; + let sql = finalize_sql_builder(sql_builder, offset, limit)?; let transfers = conn .prepare(&sql)? .query_map([], transfer_history_from_row)? @@ -859,7 +1033,17 @@ impl NftTransferHistoryStorageOps for SqliteNftStorage { let sql_transaction = conn.transaction()?; for transfer in transfers { - let transfer_json = json::to_string(&transfer).expect("serialization should not fail"); + let details_json = TransferDetailsJson { + block_hash: transfer.common.block_hash, + transaction_index: transfer.common.transaction_index, + value: transfer.common.value, + transaction_type: transfer.common.transaction_type, + verified: transfer.common.verified, + operator: transfer.common.operator, + from_address: transfer.common.from_address, + to_address: transfer.common.from_address, + }; + let transfer_json = json::to_string(&details_json).expect("serialization should not fail"); let params = [ Some(transfer.common.transaction_hash), Some(transfer.common.log_index.to_string()), @@ -872,8 +1056,10 @@ impl NftTransferHistoryStorageOps for SqliteNftStorage { Some(transfer.status.to_string()), Some(transfer.common.amount.to_string()), transfer.token_uri, + transfer.token_domain, transfer.collection_name, transfer.image_url, + transfer.image_domain, transfer.token_name, Some(i32::from(transfer.common.possible_spam).to_string()), Some(i32::from(transfer.common.possible_phishing).to_string()), @@ -909,8 +1095,10 @@ impl NftTransferHistoryStorageOps for SqliteNftStorage { let chain = *chain; async_blocking(move || { let conn = selfi.0.lock().unwrap(); - let sql_builder = get_transfers_from_block_builder(&conn, &chain, from_block)?; - let transfers = sql_builder.query(transfer_history_from_row)?; + let mut stmt = get_transfers_from_block_statement(&conn, &chain)?; + let transfers = stmt + .query_map([from_block], transfer_history_from_row)? + .collect::, _>>()?; Ok(transfers) }) .await @@ -956,21 +1144,21 @@ impl NftTransferHistoryStorageOps for SqliteNftStorage { .await } - async fn update_transfer_meta_by_hash_and_log_index( + async fn update_transfers_meta_by_token_addr_id( &self, chain: &Chain, - transfer: NftTransferHistory, + transfer_meta: TransferMeta, ) -> MmResult<(), Self::Error> { - let sql = update_meta_by_tx_hash_and_log_index_sql(chain)?; - let transfer_json = json::to_string(&transfer).expect("serialization should not fail"); + let sql = update_transfers_meta_by_token_addr_id_sql(chain)?; let params = [ - transfer.token_uri, - transfer.collection_name, - transfer.image_url, - transfer.token_name, - Some(transfer_json), - Some(transfer.common.transaction_hash), - Some(transfer.common.log_index.to_string()), + transfer_meta.token_uri, + transfer_meta.token_domain, + transfer_meta.collection_name, + transfer_meta.image_url, + transfer_meta.image_domain, + transfer_meta.token_name, + Some(transfer_meta.token_address), + Some(transfer_meta.token_id.to_string()), ]; let selfi = self.clone(); async_blocking(move || { @@ -983,28 +1171,6 @@ impl NftTransferHistoryStorageOps for SqliteNftStorage { .await } - async fn update_transfers_meta_by_token_addr_id( - &self, - chain: &Chain, - transfer_meta: TransferMeta, - ) -> MmResult<(), Self::Error> { - let selfi = self.clone(); - let transfers = selfi - .get_transfers_by_token_addr_id(chain, transfer_meta.token_address, transfer_meta.token_id) - .await?; - for mut transfer in transfers.into_iter() { - transfer.token_uri = transfer_meta.token_uri.clone(); - transfer.collection_name = transfer_meta.collection_name.clone(); - transfer.image_url = transfer_meta.image_url.clone(); - transfer.token_name = transfer_meta.token_name.clone(); - drop_mutability!(transfer); - selfi - .update_transfer_meta_by_hash_and_log_index(chain, transfer) - .await?; - } - Ok(()) - } - async fn get_transfers_with_empty_meta(&self, chain: &Chain) -> MmResult, Self::Error> { let selfi = self.clone(); let chain = *chain; @@ -1042,30 +1208,16 @@ impl NftTransferHistoryStorageOps for SqliteNftStorage { possible_spam: bool, ) -> MmResult<(), Self::Error> { let selfi = self.clone(); - let transfers = selfi.get_transfers_by_token_address(chain, token_address).await?; let table_name = nft_transfer_history_table_name(chain); validate_table_name(&table_name)?; - let sql = format!( - "UPDATE {} SET possible_spam = ?1, details_json = ?2 WHERE transaction_hash = ?3 AND log_index = ?4;", - table_name - ); + let sql = format!("UPDATE {} SET possible_spam = ?1 WHERE token_address = ?2;", table_name); async_blocking(move || { let mut conn = selfi.0.lock().unwrap(); let sql_transaction = conn.transaction()?; - - for mut transfer in transfers.into_iter() { - transfer.common.possible_spam = possible_spam; - let transfer_json = json::to_string(&transfer).expect("serialization should not fail"); - let params = [ - Some(i32::from(possible_spam).to_string()), - Some(transfer_json), - Some(transfer.common.transaction_hash.to_string()), - Some(transfer.common.log_index.to_string()), - ]; - sql_transaction.execute(&sql, params)?; - } + let params = [Some(i32::from(possible_spam).to_string()), Some(token_address.clone())]; + sql_transaction.execute(&sql, params)?; sql_transaction.commit()?; Ok(()) }) diff --git a/mm2src/coins/nft/storage/wasm/wasm_storage.rs b/mm2src/coins/nft/storage/wasm/wasm_storage.rs index bf679c0b2e..27a9d51694 100644 --- a/mm2src/coins/nft/storage/wasm/wasm_storage.rs +++ b/mm2src/coins/nft/storage/wasm/wasm_storage.rs @@ -487,25 +487,6 @@ impl NftTransferHistoryStorageOps for IndexedDbNftStorage { } } - async fn update_transfer_meta_by_hash_and_log_index( - &self, - chain: &Chain, - transfer: NftTransferHistory, - ) -> MmResult<(), Self::Error> { - let locked_db = self.lock_db().await?; - let db_transaction = locked_db.get_inner().transaction().await?; - let table = db_transaction.table::().await?; - - let index_keys = MultiIndex::new(NftTransferHistoryTable::CHAIN_TX_HASH_LOG_INDEX_INDEX) - .with_value(chain.to_string())? - .with_value(&transfer.common.transaction_hash)? - .with_value(transfer.common.log_index)?; - - let item = NftTransferHistoryTable::from_transfer_history(&transfer)?; - table.replace_item_by_unique_multi_index(index_keys, &item).await?; - Ok(()) - } - async fn update_transfers_meta_by_token_addr_id( &self, chain: &Chain, @@ -519,8 +500,10 @@ impl NftTransferHistoryStorageOps for IndexedDbNftStorage { let table = db_transaction.table::().await?; for mut transfer in transfers { transfer.token_uri = transfer_meta.token_uri.clone(); + transfer.token_domain = transfer_meta.token_domain.clone(); transfer.collection_name = transfer_meta.collection_name.clone(); transfer.image_url = transfer_meta.image_url.clone(); + transfer.image_domain = transfer_meta.image_domain.clone(); transfer.token_name = transfer_meta.token_name.clone(); drop_mutability!(transfer); From 79a468c70a98141ba2a34a6b570f54db8348e3bb Mon Sep 17 00:00:00 2001 From: laruh Date: Fri, 1 Sep 2023 19:32:38 +0700 Subject: [PATCH 04/39] add `path_to_address` --- mm2src/coins/nft.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index d928cba298..445a4842f1 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -242,6 +242,7 @@ where let mut scan_wallet_uri = prepare_uri_for_blocklist_endpoint(url_antispam, BLOCKLIST_WALLET, &chain.to_string())?; let req = MyAddressReq { coin: chain.to_ticker(), + path_to_address: Default::default(), }; let my_address = get_my_address(ctx.clone(), req).await?.wallet_address.to_lowercase(); scan_wallet_uri From e3526c356cc7c5202f6ff0e776444d13ec662a3c Mon Sep 17 00:00:00 2001 From: laruh Date: Fri, 1 Sep 2023 21:08:02 +0700 Subject: [PATCH 05/39] split antispam tests --- mm2src/coins/nft.rs | 2 ++ mm2src/coins/nft/nft_tests.rs | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index 445a4842f1..0f52149ffe 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -214,6 +214,7 @@ where let req_spam_json = serde_json::to_string(&req_spam)?; let scan_contract_res = send_post_request_to_uri(scan_contract_uri.as_str(), req_spam_json).await?; let spam_res: SpamContractRes = serde_json::from_slice(&scan_contract_res)?; + println!("spam_res {:?}", spam_res); for (address, is_spam) in spam_res.result.into_iter() { if is_spam { let address_hex = eth_addr_to_hex(&address); @@ -251,6 +252,7 @@ where .push(&my_address); let response = send_request_to_uri(scan_wallet_uri.as_str()).await?; let mnemonichq_res: MnemonicHQRes = serde_json::from_value(response)?; + println!("mnemonichq_res {:?}", mnemonichq_res); for contract in mnemonichq_res.spam_contracts.iter() { storage .update_nft_spam_by_token_address(chain, eth_addr_to_hex(contract), true) diff --git a/mm2src/coins/nft/nft_tests.rs b/mm2src/coins/nft/nft_tests.rs index 3534d5cb3e..54cc50ef8d 100644 --- a/mm2src/coins/nft/nft_tests.rs +++ b/mm2src/coins/nft/nft_tests.rs @@ -218,14 +218,17 @@ mod wasm_tests { } #[wasm_bindgen_test] - async fn test_antispam_api_requests() { + async fn test_antispam_wallet_endpoint() { let res_value = send_request_to_uri(BLOCKLIST_WALLET_ENDPOINT).await.unwrap(); let mnemonichq_res: MnemonicHQRes = serde_json::from_value(res_value).unwrap(); assert!(mnemonichq_res .spam_contracts .contains(&Address::from_str("0x0ded8542fc8b2b4e781b96e99fee6406550c9b7c").unwrap())); assert_eq!(Chain::Eth, mnemonichq_res.network); + } + #[wasm_bindgen_test] + async fn test_antispam_scan_endpoints() { let req_spam = SpamContractReq { network: Chain::Eth, addresses: "0x0ded8542fc8b2b4e781b96e99fee6406550c9b7c,0x8d1355b65da254f2cc4611453adfa8b7a13f60ee" From 7d55445cbb6fb2619dddf0227e9ca4aa037a8cb5 Mon Sep 17 00:00:00 2001 From: laruh Date: Mon, 4 Sep 2023 14:06:41 +0700 Subject: [PATCH 06/39] exclude spam phishing impls --- mm2src/coins/nft.rs | 63 ++++++++++----- mm2src/coins/nft/nft_errors.rs | 13 ++- mm2src/coins/nft/nft_structs.rs | 13 ++- mm2src/coins/nft/nft_tests.rs | 16 +++- mm2src/coins/nft/storage/db_test_helpers.rs | 81 ++++++++++++------- mm2src/coins/nft/storage/mod.rs | 5 +- mm2src/coins/nft/storage/sql_storage.rs | 44 +++++++--- mm2src/coins/nft/storage/wasm/wasm_storage.rs | 59 ++++++++++++-- 8 files changed, 215 insertions(+), 79 deletions(-) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index 0f52149ffe..e884dd400a 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -19,6 +19,7 @@ use crate::nft::nft_errors::{ProtectFromSpamError, UpdateSpamPhishingError}; use crate::nft::nft_structs::{MnemonicHQRes, NftCommon, NftCtx, NftTransferCommon, RefreshMetadataReq, SpamContractReq, SpamContractRes, TransferMeta, TransferStatus, UriMeta}; use crate::nft::storage::{NftListStorageOps, NftStorageBuilder, NftTransferHistoryStorageOps}; +use common::log::debug; use common::{parse_rfc3339_to_timestamp, APPLICATION_JSON}; use crypto::StandardHDCoinAddress; use ethereum_types::Address; @@ -64,7 +65,7 @@ pub async fn get_nft_list(ctx: MmArc, req: NftListReq) -> MmResult MmResult<(), UpdateNft Ok(()) } -/// `update_spam_phishing` function updates spam contracts info in NFT list and NFT transfers. +/// `update_spam` function updates spam contracts info in NFT list and NFT transfers. async fn update_spam( ctx: &MmArc, storage: &T, @@ -198,23 +199,16 @@ where T: NftListStorageOps + NftTransferHistoryStorageOps, { if chain == Chain::Eth || chain == Chain::Polygon { - update_spam_nft_with_mnemonichq(ctx, storage, &chain, url_antispam).await?; + return update_spam_nft_with_mnemonichq(ctx, storage, &chain, url_antispam).await; } - let scan_contract_uri = prepare_uri_for_blocklist_endpoint(url_antispam, BLOCKLIST_CONTRACT, BLOCKLIST_SCAN)?; let token_addresses = storage.get_token_addresses(&chain).await?; let addresses = token_addresses .iter() .map(eth_addr_to_hex) .collect::>() .join(","); - let req_spam = SpamContractReq { - network: chain, - addresses, - }; - let req_spam_json = serde_json::to_string(&req_spam)?; - let scan_contract_res = send_post_request_to_uri(scan_contract_uri.as_str(), req_spam_json).await?; - let spam_res: SpamContractRes = serde_json::from_slice(&scan_contract_res)?; - println!("spam_res {:?}", spam_res); + let spam_res = send_spam_request(&chain, url_antispam, addresses).await?; + debug!("\n spam_res {:?} \n", spam_res); for (address, is_spam) in spam_res.result.into_iter() { if is_spam { let address_hex = eth_addr_to_hex(&address); @@ -252,7 +246,7 @@ where .push(&my_address); let response = send_request_to_uri(scan_wallet_uri.as_str()).await?; let mnemonichq_res: MnemonicHQRes = serde_json::from_value(response)?; - println!("mnemonichq_res {:?}", mnemonichq_res); + debug!("\n mnemonichq_res {:?} \n", mnemonichq_res); for contract in mnemonichq_res.spam_contracts.iter() { storage .update_nft_spam_by_token_address(chain, eth_addr_to_hex(contract), true) @@ -264,6 +258,22 @@ where Ok(()) } +async fn send_spam_request( + chain: &Chain, + url_antispam: &Url, + addresses: String, +) -> MmResult { + let scan_contract_uri = prepare_uri_for_blocklist_endpoint(url_antispam, BLOCKLIST_CONTRACT, BLOCKLIST_SCAN)?; + let req_spam = SpamContractReq { + network: *chain, + addresses, + }; + let req_spam_json = serde_json::to_string(&req_spam)?; + let scan_contract_res = send_post_request_to_uri(scan_contract_uri.as_str(), req_spam_json).await?; + let spam_res: SpamContractRes = serde_json::from_slice(&scan_contract_res)?; + Ok(spam_res) +} + fn prepare_uri_for_blocklist_endpoint( url_antispam: &Url, blocklist_type: &str, @@ -291,13 +301,13 @@ pub async fn refresh_nft_metadata(ctx: MmArc, req: RefreshMetadataReq) -> MmResu &req.url, ) .await?; - let req = NftMetadataReq { + let req_meta = NftMetadataReq { token_address: req.token_address, token_id: req.token_id, chain: req.chain, protect_from_spam: false, }; - let mut nft_db = get_nft_metadata(ctx.clone(), req).await?; + let mut nft_db = get_nft_metadata(ctx.clone(), req_meta).await?; let token_uri = check_moralis_ipfs_bafy(moralis_meta.common.token_uri.as_deref()); let token_domain = get_domain_from_url(token_uri.as_deref()); let uri_meta = get_uri_meta(token_uri.as_deref(), moralis_meta.common.metadata.as_deref()).await; @@ -309,8 +319,22 @@ pub async fn refresh_nft_metadata(ctx: MmArc, req: RefreshMetadataReq) -> MmResu nft_db.common.last_token_uri_sync = moralis_meta.common.last_token_uri_sync; nft_db.common.last_metadata_sync = moralis_meta.common.last_metadata_sync; nft_db.common.possible_spam = moralis_meta.common.possible_spam; - // todo also update possible_phishing and check spam with antispam proxy additionally nft_db.uri_meta = uri_meta; + + let addresses = [nft_db.common.token_address] + .iter() + .map(eth_addr_to_hex) + .collect::>() + .join(","); + let spam_res = send_spam_request(&req.chain, &req.url_antispam, addresses).await?; + if let Some(true) = spam_res.result.get(&nft_db.common.token_address) { + nft_db.common.possible_spam = true; + let address_hex = eth_addr_to_hex(&nft_db.common.token_address); + storage + .update_transfer_spam_by_token_address(&req.chain, address_hex, true) + .await?; + } + // todo also update possible_phishing drop_mutability!(nft_db); storage .refresh_nft_metadata(&moralis_meta.chain, nft_db.clone()) @@ -433,7 +457,6 @@ async fn get_moralis_nft_transfers( verified: transfer_moralis.common.verified, operator: transfer_moralis.common.operator, possible_spam: transfer_moralis.common.possible_spam, - possible_phishing: false, }, chain: *chain, block_number: *transfer_moralis.block_number, @@ -446,6 +469,7 @@ async fn get_moralis_nft_transfers( image_domain: None, token_name: None, status, + possible_phishing: false, }; // collect NFTs transfers from the page res_list.push(transfer_history); @@ -526,7 +550,6 @@ async fn send_request_to_uri(uri: &str) -> MmResult { Ok(body) } -#[allow(dead_code)] async fn send_post_request_to_uri(uri: &str, body: String) -> MmResult, GetInfoFromUriError> { let (status, _header, body) = slurp_post_json(uri, body).await?; if !status.is_success() { @@ -856,12 +879,12 @@ async fn handle_receive_erc1155 for UpdateNftError { @@ -195,6 +198,10 @@ impl From for UpdateNftError { fn from(e: UpdateSpamPhishingError) -> Self { UpdateNftError::UpdateSpamPhishingError(e) } } +impl From for UpdateNftError { + fn from(e: GetInfoFromUriError) -> Self { UpdateNftError::GetInfoFromUriError(e) } +} + impl HttpStatusCode for UpdateNftError { fn status_code(&self) -> StatusCode { match self { @@ -208,13 +215,15 @@ impl HttpStatusCode for UpdateNftError { | UpdateNftError::LastScannedBlockNotFound { .. } | UpdateNftError::AttemptToReceiveAlreadyOwnedErc721 { .. } | UpdateNftError::InvalidHexString(_) - | UpdateNftError::UpdateSpamPhishingError(_) => StatusCode::INTERNAL_SERVER_ERROR, + | UpdateNftError::UpdateSpamPhishingError(_) + | UpdateNftError::GetInfoFromUriError(_) + | UpdateNftError::SerdeError(_) => StatusCode::INTERNAL_SERVER_ERROR, } } } #[derive(Clone, Debug, Deserialize, Display, EnumFromStringify, PartialEq, Serialize)] -pub(crate) enum GetInfoFromUriError { +pub enum GetInfoFromUriError { /// `http::Error` can appear on an HTTP request [`http::Builder::build`] building. #[from_stringify("http::Error")] #[display(fmt = "Invalid request: {}", _0)] diff --git a/mm2src/coins/nft/nft_structs.rs b/mm2src/coins/nft/nft_structs.rs index 2d4d467dff..55f71e56c2 100644 --- a/mm2src/coins/nft/nft_structs.rs +++ b/mm2src/coins/nft/nft_structs.rs @@ -22,7 +22,6 @@ use mm2_db::indexed_db::{ConstructibleDb, SharedDb}; #[cfg(target_arch = "wasm32")] use crate::nft::storage::wasm::nft_idb::NftCacheIDB; -#[allow(dead_code)] #[derive(Debug, Deserialize)] pub struct NftListReq { pub(crate) chains: Vec, @@ -36,7 +35,6 @@ pub struct NftListReq { pub(crate) filters: Option, } -#[allow(dead_code)] #[derive(Copy, Clone, Debug, Deserialize)] pub struct NftListFilters { #[serde(default)] @@ -60,6 +58,7 @@ pub struct RefreshMetadataReq { pub(crate) token_id: BigDecimal, pub(crate) chain: Chain, pub(crate) url: Url, + pub(crate) url_antispam: Url, } #[derive(Debug, Display)] @@ -246,8 +245,6 @@ pub struct NftCommon { pub(crate) minter_address: Option, #[serde(default)] pub(crate) possible_spam: bool, - #[serde(default)] - pub(crate) possible_phishing: bool, } #[derive(Clone, Debug, Deserialize, Serialize)] @@ -258,6 +255,8 @@ pub struct Nft { pub(crate) block_number_minted: Option, pub(crate) block_number: u64, pub(crate) contract_type: ContractType, + #[serde(default)] + pub(crate) possible_phishing: bool, pub(crate) uri_meta: UriMeta, } @@ -357,7 +356,6 @@ pub struct TransactionNftDetails { pub(crate) transaction_type: TransactionType, } -#[allow(dead_code)] #[derive(Debug, Deserialize)] pub struct NftTransfersReq { pub(crate) chains: Vec, @@ -423,8 +421,6 @@ pub struct NftTransferCommon { pub(crate) operator: Option, #[serde(default)] pub(crate) possible_spam: bool, - #[serde(default)] - pub(crate) possible_phishing: bool, } #[derive(Clone, Debug, Deserialize, Serialize)] @@ -442,6 +438,8 @@ pub struct NftTransferHistory { pub(crate) image_domain: Option, pub(crate) token_name: Option, pub(crate) status: TransferStatus, + #[serde(default)] + pub(crate) possible_phishing: bool, } /// This structure is for deserializing moralis NFT transfer json to struct. @@ -461,7 +459,6 @@ pub struct NftsTransferHistoryList { pub(crate) total: usize, } -#[allow(dead_code)] #[derive(Copy, Clone, Debug, Deserialize)] pub struct NftTransferHistoryFilters { #[serde(default)] diff --git a/mm2src/coins/nft/nft_tests.rs b/mm2src/coins/nft/nft_tests.rs index 54cc50ef8d..20dc2268eb 100644 --- a/mm2src/coins/nft/nft_tests.rs +++ b/mm2src/coins/nft/nft_tests.rs @@ -152,7 +152,10 @@ mod native_tests { fn test_nft_amount() { block_on(test_nft_amount_impl()) } #[test] - fn test_nft_spam_by_token_address() { block_on(test_update_nft_spam_by_token_address_impl()) } + fn test_update_nft_spam_by_token_address() { block_on(test_update_nft_spam_by_token_address_impl()) } + + #[test] + fn test_exclude_nft_spam() { block_on(test_exclude_nft_spam_impl()) } #[test] fn test_add_get_transfers() { block_on(test_add_get_transfers_impl()) } @@ -174,6 +177,9 @@ mod native_tests { #[test] fn test_get_token_addresses() { block_on(test_get_token_addresses_impl()) } + + #[test] + fn test_exclude_transfer_spam() { block_on(test_exclude_transfer_spam_impl()) } } #[cfg(target_arch = "wasm32")] @@ -278,7 +284,10 @@ mod wasm_tests { async fn test_refresh_metadata() { test_refresh_metadata_impl().await } #[wasm_bindgen_test] - async fn test_nft_spam_by_token_address() { test_update_nft_spam_by_token_address_impl().await } + async fn test_update_nft_spam_by_token_address() { test_update_nft_spam_by_token_address_impl().await } + + #[wasm_bindgen_test] + async fn test_exclude_nft_spam() { test_exclude_nft_spam_impl().await } #[wasm_bindgen_test] async fn test_add_get_transfers() { test_add_get_transfers_impl().await } @@ -300,4 +309,7 @@ mod wasm_tests { #[wasm_bindgen_test] async fn test_get_token_addresses() { test_get_token_addresses_impl().await } + + #[wasm_bindgen_test] + async fn test_exclude_transfer_spam() { test_exclude_transfer_spam_impl().await } } diff --git a/mm2src/coins/nft/storage/db_test_helpers.rs b/mm2src/coins/nft/storage/db_test_helpers.rs index 7b7ce13f64..e6e0c26933 100644 --- a/mm2src/coins/nft/storage/db_test_helpers.rs +++ b/mm2src/coins/nft/storage/db_test_helpers.rs @@ -1,6 +1,6 @@ use crate::eth::eth_addr_to_hex; -use crate::nft::nft_structs::{Chain, ContractType, Nft, NftCommon, NftTransferCommon, NftTransferHistory, - NftTransferHistoryFilters, TransferMeta, TransferStatus, UriMeta}; +use crate::nft::nft_structs::{Chain, ContractType, Nft, NftCommon, NftListFilters, NftTransferCommon, + NftTransferHistory, NftTransferHistoryFilters, TransferMeta, TransferStatus, UriMeta}; use crate::nft::storage::{NftListStorageOps, NftStorageBuilder, NftTransferHistoryStorageOps, RemoveNftResult}; use ethereum_types::Address; use mm2_number::BigDecimal; @@ -38,14 +38,13 @@ pub(crate) fn nft() -> Nft { last_token_uri_sync: Some("2023-02-07T17:10:08.402Z".to_string()), last_metadata_sync: Some("2023-02-07T17:10:16.858Z".to_string()), minter_address: Some("ERC1155 tokens don't have a single minter".to_string()), - possible_spam: false, - possible_phishing: false, + possible_spam: true, }, chain: Chain::Bsc, block_number_minted: Some(25465916), block_number: 25919780, contract_type: ContractType::Erc1155, - + possible_phishing: false, uri_meta: UriMeta { image_url: Some("https://tikimetadata.s3.amazonaws.com/tiki_box.png".to_string()), raw_image_url: Some("https://tikimetadata.s3.amazonaws.com/tiki_box.png".to_string()), @@ -78,13 +77,13 @@ fn nft_list() -> Vec { last_token_uri_sync: Some("2023-02-07T17:10:08.402Z".to_string()), last_metadata_sync: Some("2023-02-07T17:10:16.858Z".to_string()), minter_address: Some("ERC1155 tokens don't have a single minter".to_string()), - possible_spam: false, - possible_phishing: false, + possible_spam: true, }, chain: Chain::Bsc, block_number_minted: Some(25465916), block_number: 25919780, contract_type: ContractType::Erc1155, + possible_phishing: false, uri_meta: UriMeta { image_url: Some("https://tikimetadata.s3.amazonaws.com/tiki_box.png".to_string()), raw_image_url: None, @@ -119,13 +118,12 @@ fn nft_list() -> Vec { last_metadata_sync: Some("2023-02-16T16:36:04.283Z".to_string()), minter_address: Some("0xdbdeb0895f3681b87fb3654b5cf3e05546ba24a9".to_string()), possible_spam: false, - possible_phishing: false, }, chain: Chain::Bsc, - block_number_minted: Some(25721963), block_number: 28056726, contract_type: ContractType::Erc721, + possible_phishing: false, uri_meta: UriMeta { image_url: Some( "https://public.nftstatic.com/static/nft/res/4df0a5da04174e1e9be04b22a805f605.png".to_string(), @@ -162,13 +160,12 @@ fn nft_list() -> Vec { last_metadata_sync: Some("2023-02-16T16:36:04.283Z".to_string()), minter_address: Some("0xdbdeb0895f3681b87fb3654b5cf3e05546ba24a9".to_string()), possible_spam: false, - possible_phishing: false, }, chain: Chain::Bsc, - block_number_minted: Some(25721963), block_number: 28056726, contract_type: ContractType::Erc721, + possible_phishing: false, uri_meta: UriMeta { image_url: Some( "https://public.nftstatic.com/static/nft/res/4df0a5da04174e1e9be04b22a805f605.png".to_string(), @@ -205,13 +202,12 @@ fn nft_list() -> Vec { last_metadata_sync: Some("2023-02-19T19:12:18.080Z".to_string()), minter_address: Some("0xdbdeb0895f3681b87fb3654b5cf3e05546ba24a9".to_string()), possible_spam: false, - possible_phishing: false, }, chain: Chain::Bsc, - block_number_minted: Some(25810308), block_number: 28056721, contract_type: ContractType::Erc721, + possible_phishing: false, uri_meta: UriMeta { image_url: Some( "https://public.nftstatic.com/static/nft/res/4df0a5da04174e1e9be04b22a805f605.png".to_string(), @@ -247,8 +243,7 @@ fn nft_transfer_history() -> Vec { amount: BigDecimal::from_str("1").unwrap(), verified: Some(1), operator: Some("0x4ff0bbc9b64d635a4696d1a38554fb2529c103ff".to_string()), - possible_spam: false, - possible_phishing: false, + possible_spam: true, }, chain: Chain::Bsc, block_number: 25919780, @@ -261,6 +256,7 @@ fn nft_transfer_history() -> Vec { image_domain: None, token_name: None, status: TransferStatus::Receive, + possible_phishing: false, }; let transfer1 = NftTransferHistory { @@ -279,21 +275,19 @@ fn nft_transfer_history() -> Vec { verified: Some(1), operator: None, possible_spam: false, - possible_phishing: false, }, chain: Chain::Bsc, block_number: 28056726, block_timestamp: 1683627432, contract_type: ContractType::Erc721, - token_uri: None, token_domain: None, collection_name: None, image_url: None, image_domain: None, token_name: None, - status: TransferStatus::Receive, + possible_phishing: false, }; // Same as transfer1 but with different log_index, meaning that transfer1 and transfer2 are part of one batch/multi token transaction @@ -313,21 +307,19 @@ fn nft_transfer_history() -> Vec { verified: Some(1), operator: None, possible_spam: false, - possible_phishing: false, }, chain: Chain::Bsc, block_number: 28056726, block_timestamp: 1683627432, contract_type: ContractType::Erc721, - token_uri: None, token_domain: None, collection_name: None, image_url: None, image_domain: None, token_name: None, - status: TransferStatus::Receive, + possible_phishing: false, }; let transfer3 = NftTransferHistory { @@ -346,22 +338,19 @@ fn nft_transfer_history() -> Vec { verified: Some(1), operator: None, possible_spam: false, - possible_phishing: false, }, chain: Chain::Bsc, block_number: 28056721, block_timestamp: 1683627417, - contract_type: ContractType::Erc721, - token_uri: None, token_domain: None, collection_name: Some("Binance NFT Mystery Box-Back to Blockchain Future".to_string()), image_url: Some("https://public.nftstatic.com/static/nft/res/4df0a5da04174e1e9be04b22a805f605.png".to_string()), image_domain: None, token_name: Some("Nebula Nodes".to_string()), - status: TransferStatus::Receive, + possible_phishing: false, }; vec![transfer, transfer1, transfer2, transfer3] } @@ -421,7 +410,7 @@ pub(crate) async fn test_nft_list_impl() { storage.add_nfts_to_list(&chain, nft_list, 28056726).await.unwrap(); let nft_list = storage - .get_nft_list(vec![chain], false, 1, Some(NonZeroUsize::new(3).unwrap())) + .get_nft_list(vec![chain], false, 1, Some(NonZeroUsize::new(3).unwrap()), None) .await .unwrap(); assert_eq!(nft_list.nfts.len(), 1); @@ -444,7 +433,7 @@ pub(crate) async fn test_remove_nft_impl() { .unwrap(); assert_eq!(remove_rslt, RemoveNftResult::NftRemoved); let list_len = storage - .get_nft_list(vec![chain], true, 1, None) + .get_nft_list(vec![chain], true, 1, None, None) .await .unwrap() .nfts @@ -531,6 +520,23 @@ pub(crate) async fn test_update_nft_spam_by_token_address_impl() { } } +pub(crate) async fn test_exclude_nft_spam_impl() { + let chain = Chain::Bsc; + let storage = init_nft_list_storage(&chain).await; + let nft_list = nft_list(); + storage.add_nfts_to_list(&chain, nft_list, 28056726).await.unwrap(); + + let filters = NftListFilters { + exclude_spam: true, + exclude_phishing: false, + }; + let nft_list = storage + .get_nft_list(vec![chain], true, 1, None, Some(filters)) + .await + .unwrap(); + assert_eq!(nft_list.nfts.len(), 3); +} + pub(crate) async fn test_add_get_transfers_impl() { let chain = Chain::Bsc; let storage = init_nft_history_storage(&chain).await; @@ -706,3 +712,24 @@ pub(crate) async fn test_get_token_addresses_impl() { let token_addresses = storage.get_token_addresses(&chain).await.unwrap(); assert_eq!(token_addresses.len(), 2); } + +pub(crate) async fn test_exclude_transfer_spam_impl() { + let chain = Chain::Bsc; + let storage = init_nft_history_storage(&chain).await; + let transfers = nft_transfer_history(); + storage.add_transfers_to_history(&chain, transfers).await.unwrap(); + + let filters = NftTransferHistoryFilters { + receive: true, + send: true, + from_date: None, + to_date: None, + exclude_spam: true, + exclude_phishing: false, + }; + let transfer_history = storage + .get_transfer_history(vec![chain], true, 1, None, Some(filters)) + .await + .unwrap(); + assert_eq!(transfer_history.transfer_history.len(), 3); +} diff --git a/mm2src/coins/nft/storage/mod.rs b/mm2src/coins/nft/storage/mod.rs index 64a9460c92..d3c66e94e3 100644 --- a/mm2src/coins/nft/storage/mod.rs +++ b/mm2src/coins/nft/storage/mod.rs @@ -1,5 +1,5 @@ -use crate::nft::nft_structs::{Chain, Nft, NftList, NftTokenAddrId, NftTransferHistory, NftTransferHistoryFilters, - NftsTransferHistoryList, TransferMeta}; +use crate::nft::nft_structs::{Chain, Nft, NftList, NftListFilters, NftTokenAddrId, NftTransferHistory, + NftTransferHistoryFilters, NftsTransferHistoryList, TransferMeta}; use crate::WithdrawError; use async_trait::async_trait; use derive_more::Display; @@ -44,6 +44,7 @@ pub trait NftListStorageOps { max: bool, limit: usize, page_number: Option, + filters: Option, ) -> MmResult; async fn add_nfts_to_list(&self, chain: &Chain, nfts: I, last_scanned_block: u64) -> MmResult<(), Self::Error> diff --git a/mm2src/coins/nft/storage/sql_storage.rs b/mm2src/coins/nft/storage/sql_storage.rs index d01d09ecfd..6a85e598ca 100644 --- a/mm2src/coins/nft/storage/sql_storage.rs +++ b/mm2src/coins/nft/storage/sql_storage.rs @@ -1,6 +1,6 @@ use crate::nft::eth_addr_to_hex; -use crate::nft::nft_structs::{Chain, ContractType, ConvertChain, Nft, NftCommon, NftList, NftTokenAddrId, - NftTransferCommon, NftTransferHistory, NftTransferHistoryFilters, +use crate::nft::nft_structs::{Chain, ContractType, ConvertChain, Nft, NftCommon, NftList, NftListFilters, + NftTokenAddrId, NftTransferCommon, NftTransferHistory, NftTransferHistoryFilters, NftsTransferHistoryList, TransferMeta, UriMeta}; use crate::nft::storage::{get_offset_limit, CreateNftStorageError, NftDetailsJson, NftListStorageOps, NftStorageError, NftTransferHistoryStorageOps, RemoveNftResult, TransferDetailsJson}; @@ -128,13 +128,16 @@ impl SqliteNftStorage { } } -fn get_nft_list_builder_preimage(chains: Vec) -> MmResult { +fn get_nft_list_builder_preimage( + chains: Vec, + filters: Option, +) -> MmResult { let union_sql_strings = chains .iter() .map(|chain| { let table_name = nft_list_table_name(chain); validate_table_name(&table_name)?; - let sql_builder = SqlBuilder::select_from(table_name.as_str()); + let sql_builder = nft_list_builder_preimage(table_name.as_str(), filters)?; let sql_string = sql_builder .sql() .map_err(|e| SqlError::ToSqlConversionFailure(e.into()))? @@ -150,6 +153,20 @@ fn get_nft_list_builder_preimage(chains: Vec) -> MmResult) -> Result { + let mut sql_builder = SqlBuilder::select_from(table_name); + if let Some(filters) = filters { + if filters.exclude_spam { + sql_builder.and_where("possible_spam == 0"); + } + if filters.exclude_phishing { + sql_builder.and_where("possible_phishing == 0"); + } + } + drop_mutability!(sql_builder); + Ok(sql_builder) +} + fn get_nft_transfer_builder_preimage( chains: Vec, filters: Option, @@ -192,6 +209,12 @@ fn nft_history_table_builder_preimage( if let Some(date) = filters.to_date { sql_builder.and_where(format!("block_timestamp <= {}", date)); } + if filters.exclude_spam { + sql_builder.and_where("possible_spam == 0"); + } + if filters.exclude_phishing { + sql_builder.and_where("possible_phishing == 0"); + } } drop_mutability!(sql_builder); Ok(sql_builder) @@ -282,7 +305,6 @@ fn nft_from_row(row: &Row<'_>) -> Result { last_metadata_sync, minter_address: nft_details.minter_address, possible_spam: possible_spam != 0, - possible_phishing: possible_phishing != 0, }; let nft = Nft { common, @@ -290,6 +312,7 @@ fn nft_from_row(row: &Row<'_>) -> Result { block_number_minted: nft_details.block_number_minted, block_number, contract_type, + possible_phishing: possible_phishing != 0, uri_meta, }; Ok(nft) @@ -333,7 +356,6 @@ fn transfer_history_from_row(row: &Row<'_>) -> Result) -> Result, + filters: Option, ) -> MmResult { let selfi = self.clone(); async_blocking(move || { let conn = selfi.0.lock().unwrap(); - let sql_builder = get_nft_list_builder_preimage(chains)?; + let sql_builder = get_nft_list_builder_preimage(chains, filters)?; let total_count_builder_sql = sql_builder .clone() .count("*") @@ -722,7 +746,7 @@ impl NftListStorageOps for SqliteNftStorage { Some(nft.block_number.to_string()), Some(nft.contract_type.to_string()), Some(i32::from(nft.common.possible_spam).to_string()), - Some(i32::from(nft.common.possible_phishing).to_string()), + Some(i32::from(nft.possible_phishing).to_string()), nft.common.collection_name, nft.common.symbol, nft.common.token_uri, @@ -821,7 +845,7 @@ impl NftListStorageOps for SqliteNftStorage { let sql_transaction = conn.transaction()?; let params = [ Some(i32::from(nft.common.possible_spam).to_string()), - Some(i32::from(nft.common.possible_phishing).to_string()), + Some(i32::from(nft.possible_phishing).to_string()), nft.common.collection_name, nft.common.symbol, nft.common.token_uri, @@ -1062,7 +1086,7 @@ impl NftTransferHistoryStorageOps for SqliteNftStorage { transfer.image_domain, transfer.token_name, Some(i32::from(transfer.common.possible_spam).to_string()), - Some(i32::from(transfer.common.possible_phishing).to_string()), + Some(i32::from(transfer.possible_phishing).to_string()), Some(transfer_json), ]; sql_transaction.execute(&insert_transfer_in_history_sql(&chain)?, params)?; diff --git a/mm2src/coins/nft/storage/wasm/wasm_storage.rs b/mm2src/coins/nft/storage/wasm/wasm_storage.rs index 27a9d51694..2596ec2f6d 100644 --- a/mm2src/coins/nft/storage/wasm/wasm_storage.rs +++ b/mm2src/coins/nft/storage/wasm/wasm_storage.rs @@ -1,6 +1,6 @@ use crate::eth::eth_addr_to_hex; -use crate::nft::nft_structs::{Chain, ContractType, Nft, NftCtx, NftList, NftTransferHistory, NftsTransferHistoryList, - TransferMeta, TransferStatus}; +use crate::nft::nft_structs::{Chain, ContractType, Nft, NftCtx, NftList, NftListFilters, NftTransferHistory, + NftsTransferHistoryList, TransferMeta, TransferStatus}; use crate::nft::storage::wasm::nft_idb::{NftCacheIDB, NftCacheIDBLocked}; use crate::nft::storage::wasm::{WasmNftCacheError, WasmNftCacheResult}; use crate::nft::storage::{get_offset_limit, CreateNftStorageError, NftListStorageOps, NftTokenAddrId, @@ -53,6 +53,24 @@ impl IndexedDbNftStorage { }) } + fn take_nfts_according_to_filters(nfts: I, filters: Option) -> WasmNftCacheResult> + where + I: Iterator, + { + let mut filtered_nfts = Vec::new(); + for nft_table in nfts { + let nft = nft_details_from_item(nft_table)?; + if let Some(filters) = &filters { + if filters.is_spam_match(&nft) && filters.is_phishing_match(&nft) { + filtered_nfts.push(nft); + } + } else { + filtered_nfts.push(nft); + } + } + Ok(filtered_nfts) + } + fn take_transfers_according_to_paging_opts( mut transfers: Vec, max: bool, @@ -80,7 +98,11 @@ impl IndexedDbNftStorage { for transfers_table in transfers { let transfer = transfer_details_from_item(transfers_table)?; if let Some(filters) = &filters { - if filters.is_status_match(&transfer) && filters.is_date_match(&transfer) { + if filters.is_status_match(&transfer) + && filters.is_date_match(&transfer) + && filters.is_spam_match(&transfer) + && filters.is_phishing_match(&transfer) + { filtered_transfers.push(transfer); } } else { @@ -91,6 +113,12 @@ impl IndexedDbNftStorage { } } +impl NftListFilters { + fn is_spam_match(&self, nft: &Nft) -> bool { !self.exclude_spam || !nft.common.possible_spam } + + fn is_phishing_match(&self, nft: &Nft) -> bool { !self.exclude_phishing || !nft.possible_phishing } +} + impl NftTransferHistoryFilters { fn is_status_match(&self, transfer: &NftTransferHistory) -> bool { (!self.receive && !self.send) @@ -102,6 +130,14 @@ impl NftTransferHistoryFilters { self.from_date.map_or(true, |from| transfer.block_timestamp >= from) && self.to_date.map_or(true, |to| transfer.block_timestamp <= to) } + + fn is_spam_match(&self, transfer: &NftTransferHistory) -> bool { + !self.exclude_spam || !transfer.common.possible_spam + } + + fn is_phishing_match(&self, transfer: &NftTransferHistory) -> bool { + !self.exclude_phishing || !transfer.possible_phishing + } } #[async_trait] @@ -118,17 +154,20 @@ impl NftListStorageOps for IndexedDbNftStorage { max: bool, limit: usize, page_number: Option, + filters: Option, ) -> MmResult { let locked_db = self.lock_db().await?; let db_transaction = locked_db.get_inner().transaction().await?; let table = db_transaction.table::().await?; let mut nfts = Vec::new(); for chain in chains { - let items = table.get_items("chain", chain.to_string()).await?; - for (_item_id, item) in items.into_iter() { - let nft_detail = nft_details_from_item(item)?; - nfts.push(nft_detail); - } + let nft_tables = table + .get_items("chain", chain.to_string()) + .await? + .into_iter() + .map(|(_item_id, nft)| nft); + let filtered = Self::take_nfts_according_to_filters(nft_tables, filters)?; + nfts.extend(filtered); } Self::take_nft_according_to_paging_opts(nfts, max, limit, page_number) } @@ -668,6 +707,7 @@ pub(crate) struct NftListTable { block_number: BeBigUint, contract_type: ContractType, possible_spam: bool, + possible_phishing: bool, details_json: Json, } @@ -688,6 +728,7 @@ impl NftListTable { block_number: BeBigUint::from(nft.block_number), contract_type: nft.contract_type, possible_spam: nft.common.possible_spam, + possible_phishing: nft.possible_phishing, details_json, }) } @@ -730,6 +771,7 @@ pub(crate) struct NftTransferHistoryTable { image_url: Option, token_name: Option, possible_spam: bool, + possible_phishing: bool, details_json: Json, } @@ -761,6 +803,7 @@ impl NftTransferHistoryTable { image_url: transfer.image_url.clone(), token_name: transfer.token_name.clone(), possible_spam: transfer.common.possible_spam, + possible_phishing: transfer.possible_phishing, details_json, }) } From e1f2b1c56e907705e4504af12ec4cec80f7d018e Mon Sep 17 00:00:00 2001 From: laruh Date: Mon, 4 Sep 2023 17:07:23 +0700 Subject: [PATCH 07/39] check update_spam logs --- mm2src/coins/nft.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index e884dd400a..11f0a462ed 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -198,7 +198,9 @@ async fn update_spam( where T: NftListStorageOps + NftTransferHistoryStorageOps, { + log!("WE ARE IN update_spam \n Chain = {}", chain); if chain == Chain::Eth || chain == Chain::Polygon { + log!("WE ARE IN if chain == Chain::Eth || chain == Chain::Polygon"); return update_spam_nft_with_mnemonichq(ctx, storage, &chain, url_antispam).await; } let token_addresses = storage.get_token_addresses(&chain).await?; From e7b9162c16fc2542ba5f027fb713a7140954406e Mon Sep 17 00:00:00 2001 From: laruh Date: Mon, 4 Sep 2023 17:58:16 +0700 Subject: [PATCH 08/39] add more doc comments --- mm2src/coins/nft.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index 11f0a462ed..17b3ac87d3 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -19,7 +19,6 @@ use crate::nft::nft_errors::{ProtectFromSpamError, UpdateSpamPhishingError}; use crate::nft::nft_structs::{MnemonicHQRes, NftCommon, NftCtx, NftTransferCommon, RefreshMetadataReq, SpamContractReq, SpamContractRes, TransferMeta, TransferStatus, UriMeta}; use crate::nft::storage::{NftListStorageOps, NftStorageBuilder, NftTransferHistoryStorageOps}; -use common::log::debug; use common::{parse_rfc3339_to_timestamp, APPLICATION_JSON}; use crypto::StandardHDCoinAddress; use ethereum_types::Address; @@ -148,6 +147,7 @@ pub async fn update_nft(ctx: MmArc, req: UpdateNftReq) -> MmResult<(), UpdateNft let nfts = cache_nfts_from_moralis(&ctx, &storage, chain, &req.url).await?; update_meta_in_transfers(&storage, chain, nfts).await?; update_transfers_with_empty_meta(&storage, chain, &req.url).await?; + update_spam(&ctx, &storage, *chain, &req.url_antispam).await?; continue; }, Err(_) => { @@ -162,6 +162,7 @@ pub async fn update_nft(ctx: MmArc, req: UpdateNftReq) -> MmResult<(), UpdateNft .await?; update_meta_in_transfers(&storage, chain, nft_list).await?; update_transfers_with_empty_meta(&storage, chain, &req.url).await?; + update_spam(&ctx, &storage, *chain, &req.url_antispam).await?; continue; }, }; @@ -198,9 +199,7 @@ async fn update_spam( where T: NftListStorageOps + NftTransferHistoryStorageOps, { - log!("WE ARE IN update_spam \n Chain = {}", chain); if chain == Chain::Eth || chain == Chain::Polygon { - log!("WE ARE IN if chain == Chain::Eth || chain == Chain::Polygon"); return update_spam_nft_with_mnemonichq(ctx, storage, &chain, url_antispam).await; } let token_addresses = storage.get_token_addresses(&chain).await?; @@ -210,7 +209,6 @@ where .collect::>() .join(","); let spam_res = send_spam_request(&chain, url_antispam, addresses).await?; - debug!("\n spam_res {:?} \n", spam_res); for (address, is_spam) in spam_res.result.into_iter() { if is_spam { let address_hex = eth_addr_to_hex(&address); @@ -248,7 +246,6 @@ where .push(&my_address); let response = send_request_to_uri(scan_wallet_uri.as_str()).await?; let mnemonichq_res: MnemonicHQRes = serde_json::from_value(response)?; - debug!("\n mnemonichq_res {:?} \n", mnemonichq_res); for contract in mnemonichq_res.spam_contracts.iter() { storage .update_nft_spam_by_token_address(chain, eth_addr_to_hex(contract), true) @@ -260,6 +257,7 @@ where Ok(()) } +/// `send_spam_request` function sends request to antispam api to scan contract addresses for spam. async fn send_spam_request( chain: &Chain, url_antispam: &Url, @@ -276,6 +274,8 @@ async fn send_spam_request( Ok(spam_res) } +/// `prepare_uri_for_blocklist_endpoint` function constructs the URI required for the antispam API request. +/// It appends the required path segments to the given base URL and returns the completed URI. fn prepare_uri_for_blocklist_endpoint( url_antispam: &Url, blocklist_type: &str, From f5a142b1421e79056dbfb20b1231c31dc5a1f247 Mon Sep 17 00:00:00 2001 From: laruh Date: Tue, 5 Sep 2023 20:24:15 +0700 Subject: [PATCH 09/39] doc comments --- mm2src/coins/nft.rs | 51 +++++++++++++++++++++++++++++++-- mm2src/coins/nft/nft_errors.rs | 16 +++++++++++ mm2src/coins/nft/nft_structs.rs | 51 +++++++++++++++++++++++++++++---- 3 files changed, 109 insertions(+), 9 deletions(-) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index 17b3ac87d3..d5acc4c1f7 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -50,6 +50,8 @@ const BLOCKLIST_DOMAIN: &str = "domain"; const BLOCKLIST_WALLET: &str = "wallet"; const BLOCKLIST_SCAN: &str = "scan"; +/// `WithdrawNftResult` type represents the result of an NFT withdrawal operation. On success, it provides the details +/// of the generated transaction meant for transferring the NFT. On failure, it details the encountered error. pub type WithdrawNftResult = Result>; /// `get_nft_list` function returns list of NFTs on requested chains owned by user. @@ -75,7 +77,20 @@ pub async fn get_nft_list(ctx: MmArc, req: NftListReq) -> MmResult`: Result containing the desired NFT or an error. pub async fn get_nft_metadata(ctx: MmArc, req: NftMetadataReq) -> MmResult { let nft_ctx = NftCtx::from_ctx(&ctx).map_to_mm(GetNftInfoError::Internal)?; let _lock = nft_ctx.guard.lock().await; @@ -121,7 +136,20 @@ pub async fn get_nft_transfers(ctx: MmArc, req: NftTransfersReq) -> MmResult`: A result indicating success or an error. pub async fn update_nft(ctx: MmArc, req: UpdateNftReq) -> MmResult<(), UpdateNftError> { let nft_ctx = NftCtx::from_ctx(&ctx).map_to_mm(GetNftInfoError::Internal)?; let _lock = nft_ctx.guard.lock().await; @@ -290,7 +318,22 @@ fn prepare_uri_for_blocklist_endpoint( Ok(uri) } -/// `refresh_nft_metadata` function refreshes metadata related to NFT with specified token address and token id. +/// Refreshes and updates metadata associated with a specific NFT. +/// +/// The function obtains updated metadata for an NFT using its token address and token id. +/// It fetches the metadata from the provided `url` and validates it against possible spam and +/// phishing domains using the provided `url_antispam`. If the fetched metadata or its domain +/// is identified as spam or matches with any phishing domains, the NFT's `possible_spam` and/or +/// `possible_phishing` flags are set to true. +/// +/// # Arguments +/// +/// * `ctx`: Context required for handling internal operations. +/// * `req`: A request containing details about the NFT whose metadata needs to be refreshed. +/// +/// # Returns +/// +/// * `MmResult<(), UpdateNftError>`: A result indicating success or an error. pub async fn refresh_nft_metadata(ctx: MmArc, req: RefreshMetadataReq) -> MmResult<(), UpdateNftError> { let nft_ctx = NftCtx::from_ctx(&ctx).map_to_mm(GetNftInfoError::Internal)?; let _lock = nft_ctx.guard.lock().await; @@ -1013,6 +1056,7 @@ fn protect_from_history_spam(transfer: &mut NftTransferHistory) -> MmResult<(), /// `collection_name` and `token_name` in `Nft` shouldn't contain any links, /// they must be just an arbitrary text, which represents NFT names. /// `symbol` also must be a text or sign that represents a symbol. +/// This function also checks `metadata` field for spam. fn protect_from_nft_spam(nft: &mut Nft) -> MmResult<(), ProtectFromSpamError> { let collection_name_spam = check_and_redact_if_spam(&mut nft.common.collection_name)?; let symbol_spam = check_and_redact_if_spam(&mut nft.common.symbol)?; @@ -1024,6 +1068,7 @@ fn protect_from_nft_spam(nft: &mut Nft) -> MmResult<(), ProtectFromSpamError> { } Ok(()) } + /// `check_nft_metadata_for_spam` function checks and redact spam in `metadata` field from `Nft`. /// /// **note:** `token_name` is usually called `name` in `metadata`. diff --git a/mm2src/coins/nft/nft_errors.rs b/mm2src/coins/nft/nft_errors.rs index 37a9346bff..47f7337a36 100644 --- a/mm2src/coins/nft/nft_errors.rs +++ b/mm2src/coins/nft/nft_errors.rs @@ -9,6 +9,7 @@ use mm2_net::transport::SlurpError; use serde::{Deserialize, Serialize}; use web3::Error; +/// Enumerates potential errors that can arise when fetching NFT information. #[derive(Clone, Debug, Deserialize, Display, EnumFromStringify, PartialEq, Serialize, SerializeErrorType)] #[serde(tag = "error_type", content = "error_data")] pub enum GetNftInfoError { @@ -119,6 +120,16 @@ impl HttpStatusCode for GetNftInfoError { } } +/// Enumerates possible errors that can occur while updating NFT details in the database. +/// +/// The errors capture various issues that can arise during: +/// - Metadata refresh +/// - NFT transfer history updating +/// - NFT list updating +/// +/// The issues addressed include database errors, invalid hex strings, +/// inconsistencies in block numbers, and problems related to fetching or interpreting +/// fetched metadata. #[derive(Clone, Debug, Deserialize, Display, EnumFromStringify, PartialEq, Serialize, SerializeErrorType)] #[serde(tag = "error_type", content = "error_data")] pub enum UpdateNftError { @@ -291,3 +302,8 @@ impl From for UpdateSpamPhishingError { impl From for UpdateSpamPhishingError { fn from(err: T) -> Self { UpdateSpamPhishingError::DbError(format!("{:?}", err)) } } + +#[derive(Debug, Display)] +pub enum ParseChainTypeError { + UnsupportedChainType, +} diff --git a/mm2src/coins/nft/nft_structs.rs b/mm2src/coins/nft/nft_structs.rs index 55f71e56c2..66509be9e9 100644 --- a/mm2src/coins/nft/nft_structs.rs +++ b/mm2src/coins/nft/nft_structs.rs @@ -16,33 +16,55 @@ use std::str::FromStr; use std::sync::Arc; use url::Url; +use crate::nft::nft_errors::ParseChainTypeError; #[cfg(target_arch = "wasm32")] use mm2_db::indexed_db::{ConstructibleDb, SharedDb}; #[cfg(target_arch = "wasm32")] use crate::nft::storage::wasm::nft_idb::NftCacheIDB; +/// Represents a request to list NFTs owned by the user across specified chains. +/// +/// The request provides options such as pagination, limiting the number of results, +/// and applying specific filters to the list. #[derive(Debug, Deserialize)] pub struct NftListReq { + /// List of chains to fetch the NFTs from. pub(crate) chains: Vec, + /// Parameter indicating if the maximum number of NFTs should be fetched. + /// If true, then `limit` will be ignored. #[serde(default)] pub(crate) max: bool, + /// Limit to the number of NFTs returned in a single request. #[serde(default = "ten")] pub(crate) limit: usize, + /// Page number for pagination. pub(crate) page_number: Option, + /// Flag indicating if the returned list should be protected from potential spam. #[serde(default)] pub(crate) protect_from_spam: bool, + /// Optional filters to apply when listing the NFTs. pub(crate) filters: Option, } +/// Filters that can be applied when listing NFTs to exclude potential threats or nuisances. #[derive(Copy, Clone, Debug, Deserialize)] pub struct NftListFilters { + /// Exclude NFTs that are flagged as possible spam. #[serde(default)] pub(crate) exclude_spam: bool, + /// Exclude NFTs that are flagged as phishing attempts. #[serde(default)] pub(crate) exclude_phishing: bool, } +/// Contains parameters required to fetch metadata for a specified NFT. +/// # Fields +/// * `token_address`: The address of the NFT token. +/// * `token_id`: The ID of the NFT token. +/// * `chain`: The blockchain chain where the NFT exists. +/// * `protect_from_spam`: Indicates whether to check and redact potential spam. If set to true, +/// the internal function `protect_from_nft_spam` is utilized. #[derive(Debug, Deserialize)] pub struct NftMetadataReq { pub(crate) token_address: Address, @@ -52,6 +74,14 @@ pub struct NftMetadataReq { pub(crate) protect_from_spam: bool, } +/// Contains parameters required to refresh metadata for a specified NFT. +/// # Fields +/// * `token_address`: The address of the NFT token whose metadata needs to be refreshed. +/// * `token_id`: The ID of the NFT token. +/// * `chain`: The blockchain chain where the NFT exists. +/// * `url`: URL to fetch the metadata. +/// * `url_antispam`: URL used to validate if the fetched contract addresses are associated +/// with spam contracts or if domain fields in the fetched metadata match known phishing domains. #[derive(Debug, Deserialize)] pub struct RefreshMetadataReq { pub(crate) token_address: Address, @@ -61,11 +91,8 @@ pub struct RefreshMetadataReq { pub(crate) url_antispam: Url, } -#[derive(Debug, Display)] -pub enum ParseChainTypeError { - UnsupportedChainType, -} - +/// Represents blockchains which are supported by NFT feature. +/// Currently there are only EVM based chains. #[derive(Clone, Copy, Debug, PartialEq, Serialize)] #[serde(rename_all = "UPPERCASE")] pub enum Chain { @@ -247,6 +274,8 @@ pub struct NftCommon { pub(crate) possible_spam: bool, } +/// Represents an NFT with specific chain details, contract type, and other relevant attributes. +/// This structure captures detailed information about an NFT. #[derive(Clone, Debug, Deserialize, Serialize)] pub struct Nft { #[serde(flatten)] @@ -260,7 +289,9 @@ pub struct Nft { pub(crate) uri_meta: UriMeta, } -/// This structure is for deserializing moralis NFT json to struct. +/// Represents an NFT structure specifically for deserialization from Moralis's JSON response. +/// +/// This structure is adapted to the specific format provided by Moralis's API. #[derive(Debug, Deserialize)] pub(crate) struct NftFromMoralis { #[serde(flatten)] @@ -293,6 +324,8 @@ impl std::ops::Deref for SerdeStringWrap { fn deref(&self) -> &T { &self.0 } } +/// Represents a detailed list of NFTs, including the total number of NFTs and the number of skipped NFTs. +/// It is used as response of `get_nft_list` if it is successful. #[derive(Debug, Serialize)] pub struct NftList { pub(crate) nfts: Vec, @@ -473,6 +506,12 @@ pub struct NftTransferHistoryFilters { pub(crate) exclude_phishing: bool, } +/// Contains parameters required to update NFT transfer history and NFT list. +/// # Fields +/// * `chains`: A list of blockchain chains for which the NFTs need to be updated. +/// * `url`: URL to fetch the NFT data. +/// * `url_antispam`: URL used to validate if the fetched contract addresses are associated +/// with spam contracts or if domain fields in the fetched metadata match known phishing domains. #[derive(Debug, Deserialize)] pub struct UpdateNftReq { pub(crate) chains: Vec, From 0ba06ef60defc867b2ea6d112129baaeba161b6a Mon Sep 17 00:00:00 2001 From: laruh Date: Wed, 6 Sep 2023 15:06:43 +0700 Subject: [PATCH 10/39] impl `to_table_name` fncs, `filter_nfts` and `filter_transfers`,use HashSet, passes filter funcs, lock_db at the beginning --- mm2src/coins/nft.rs | 2 +- mm2src/coins/nft/storage/db_test_helpers.rs | 2 +- mm2src/coins/nft/storage/mod.rs | 5 +- mm2src/coins/nft/storage/sql_storage.rs | 201 ++++++------------ mm2src/coins/nft/storage/wasm/wasm_storage.rs | 34 +-- 5 files changed, 92 insertions(+), 152 deletions(-) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index d5acc4c1f7..bdb3df043c 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -1005,7 +1005,7 @@ async fn update_transfers_with_empty_meta(storage: &T, chain: &Chain, url: &U where T: NftListStorageOps + NftTransferHistoryStorageOps, { - let nft_token_addr_id = storage.get_transfers_with_empty_meta(chain).await?; + let nft_token_addr_id = storage.get_transfers_with_empty_meta(*chain).await?; for addr_id_pair in nft_token_addr_id.into_iter() { let nft_meta = get_moralis_metadata(addr_id_pair.token_address, addr_id_pair.token_id, chain, url).await?; let transfer_meta = TransferMeta::from(nft_meta); diff --git a/mm2src/coins/nft/storage/db_test_helpers.rs b/mm2src/coins/nft/storage/db_test_helpers.rs index e6e0c26933..0a0a388b2c 100644 --- a/mm2src/coins/nft/storage/db_test_helpers.rs +++ b/mm2src/coins/nft/storage/db_test_helpers.rs @@ -658,7 +658,7 @@ pub(crate) async fn test_get_update_transfer_meta_impl() { let transfers = nft_transfer_history(); storage.add_transfers_to_history(&chain, transfers).await.unwrap(); - let vec_token_add_id = storage.get_transfers_with_empty_meta(&chain).await.unwrap(); + let vec_token_add_id = storage.get_transfers_with_empty_meta(chain).await.unwrap(); assert_eq!(vec_token_add_id.len(), 3); let token_add = "0x5c7d6712dfaf0cb079d48981781c8705e8417ca0".to_string(); diff --git a/mm2src/coins/nft/storage/mod.rs b/mm2src/coins/nft/storage/mod.rs index d3c66e94e3..a9063171a2 100644 --- a/mm2src/coins/nft/storage/mod.rs +++ b/mm2src/coins/nft/storage/mod.rs @@ -9,6 +9,7 @@ use mm2_err_handle::mm_error::MmResult; use mm2_err_handle::mm_error::{NotEqual, NotMmError}; use mm2_number::BigDecimal; use serde::{Deserialize, Serialize}; +use std::collections::HashSet; use std::num::NonZeroUsize; #[cfg(any(test, target_arch = "wasm32"))] @@ -154,7 +155,7 @@ pub trait NftTransferHistoryStorageOps { transfer_meta: TransferMeta, ) -> MmResult<(), Self::Error>; - async fn get_transfers_with_empty_meta(&self, chain: &Chain) -> MmResult, Self::Error>; + async fn get_transfers_with_empty_meta(&self, chain: Chain) -> MmResult, Self::Error>; /// `get_transfers_by_token_address` function returns list of NFT transfers which have specified token address. async fn get_transfers_by_token_address( @@ -172,7 +173,7 @@ pub trait NftTransferHistoryStorageOps { ) -> MmResult<(), Self::Error>; /// `get_token_addresses` return all unique token addresses. - async fn get_token_addresses(&self, chain: &Chain) -> MmResult, Self::Error>; + async fn get_token_addresses(&self, chain: &Chain) -> MmResult, Self::Error>; } #[derive(Debug, Deserialize, Display, Serialize)] diff --git a/mm2src/coins/nft/storage/sql_storage.rs b/mm2src/coins/nft/storage/sql_storage.rs index 6a85e598ca..976972b26b 100644 --- a/mm2src/coins/nft/storage/sql_storage.rs +++ b/mm2src/coins/nft/storage/sql_storage.rs @@ -8,7 +8,7 @@ use async_trait::async_trait; use common::async_blocking; use db_common::sql_build::{SqlCondition, SqlQuery}; use db_common::sqlite::rusqlite::types::{FromSqlError, Type}; -use db_common::sqlite::rusqlite::{Connection, Error as SqlError, Row, Statement}; +use db_common::sqlite::rusqlite::{Connection, Error as SqlError, Result as SqlResult, Row, Statement}; use db_common::sqlite::sql_builder::SqlBuilder; use db_common::sqlite::{query_single_row, string_from_row, validate_table_name, CHECK_TABLE_EXISTS_SQL}; use ethereum_types::Address; @@ -18,20 +18,34 @@ use mm2_err_handle::mm_error::{MmError, MmResult}; use mm2_number::BigDecimal; use serde_json::Value as Json; use serde_json::{self as json}; +use std::collections::HashSet; use std::convert::TryInto; use std::num::NonZeroUsize; use std::str::FromStr; use std::sync::{Arc, Mutex}; -fn nft_list_table_name(chain: &Chain) -> String { chain.to_ticker() + "_nft_list" } +impl Chain { + fn to_nft_list_table_name(self) -> SqlResult { + let name = self.to_ticker() + "_nft_list"; + validate_table_name(&name)?; + Ok(name) + } -fn nft_transfer_history_table_name(chain: &Chain) -> String { chain.to_ticker() + "_nft_transfer_history" } + fn to_transfer_history_table_name(self) -> SqlResult { + let name = self.to_ticker() + "_nft_transfer_history"; + validate_table_name(&name)?; + Ok(name) + } +} -fn scanned_nft_blocks_table_name() -> String { "scanned_nft_blocks".to_string() } +fn scanned_nft_blocks_table_name() -> SqlResult { + let name = "scanned_nft_blocks".to_string(); + validate_table_name(&name)?; + Ok(name) +} fn create_nft_list_table_sql(chain: &Chain) -> MmResult { - let table_name = nft_list_table_name(chain); - validate_table_name(&table_name)?; + let table_name = chain.to_nft_list_table_name()?; let sql = format!( "CREATE TABLE IF NOT EXISTS {} ( token_address VARCHAR(256) NOT NULL, @@ -69,8 +83,7 @@ fn create_nft_list_table_sql(chain: &Chain) -> MmResult { } fn create_transfer_history_table_sql(chain: &Chain) -> MmResult { - let table_name = nft_transfer_history_table_name(chain); - validate_table_name(&table_name)?; + let table_name = chain.to_transfer_history_table_name()?; let sql = format!( "CREATE TABLE IF NOT EXISTS {} ( transaction_hash VARCHAR(256) NOT NULL, @@ -100,8 +113,7 @@ fn create_transfer_history_table_sql(chain: &Chain) -> MmResult MmResult { - let table_name = scanned_nft_blocks_table_name(); - validate_table_name(&table_name)?; + let table_name = scanned_nft_blocks_table_name()?; let sql = format!( "CREATE TABLE IF NOT EXISTS {} ( chain TEXT PRIMARY KEY, @@ -135,8 +147,7 @@ fn get_nft_list_builder_preimage( let union_sql_strings = chains .iter() .map(|chain| { - let table_name = nft_list_table_name(chain); - validate_table_name(&table_name)?; + let table_name = chain.to_nft_list_table_name()?; let sql_builder = nft_list_builder_preimage(table_name.as_str(), filters)?; let sql_string = sql_builder .sql() @@ -174,8 +185,7 @@ fn get_nft_transfer_builder_preimage( let union_sql_strings = chains .into_iter() .map(|chain| { - let table_name = nft_transfer_history_table_name(&chain); - validate_table_name(&table_name)?; + let table_name = chain.to_transfer_history_table_name()?; let sql_builder = nft_history_table_builder_preimage(table_name.as_str(), filters)?; let sql_string = sql_builder .sql() @@ -395,9 +405,7 @@ fn token_address_id_from_row(row: &Row<'_>) -> Result } fn insert_nft_in_list_sql(chain: &Chain) -> MmResult { - let table_name = nft_list_table_name(chain); - validate_table_name(&table_name)?; - + let table_name = chain.to_nft_list_table_name()?; let sql = format!( "INSERT INTO {} ( token_address, token_id, chain, amount, block_number, contract_type, possible_spam, @@ -415,9 +423,7 @@ fn insert_nft_in_list_sql(chain: &Chain) -> MmResult { } fn insert_transfer_in_history_sql(chain: &Chain) -> MmResult { - let table_name = nft_transfer_history_table_name(chain); - validate_table_name(&table_name)?; - + let table_name = chain.to_transfer_history_table_name()?; let sql = format!( "INSERT INTO {} ( transaction_hash, log_index, chain, block_number, block_timestamp, contract_type, @@ -432,8 +438,7 @@ fn insert_transfer_in_history_sql(chain: &Chain) -> MmResult { } fn upsert_last_scanned_block_sql() -> MmResult { - let table_name = scanned_nft_blocks_table_name(); - validate_table_name(&table_name)?; + let table_name = scanned_nft_blocks_table_name()?; let sql = format!( "INSERT OR REPLACE INTO {} (chain, last_scanned_block) VALUES (?1, ?2);", table_name @@ -442,9 +447,7 @@ fn upsert_last_scanned_block_sql() -> MmResult { } fn refresh_nft_metadata_sql(chain: &Chain) -> MmResult { - let table_name = nft_list_table_name(chain); - - validate_table_name(&table_name)?; + let table_name = chain.to_nft_list_table_name()?; let sql = format!( "UPDATE {} SET possible_spam = ?1, possible_phishing = ?2, collection_name = ?3, symbol = ?4, token_uri = ?5, token_domain = ?6, metadata = ?7, \ last_token_uri_sync = ?8, last_metadata_sync = ?9, raw_image_url = ?10, image_url = ?11, image_domain = ?12, token_name = ?13, description = ?14, \ @@ -455,9 +458,7 @@ fn refresh_nft_metadata_sql(chain: &Chain) -> MmResult { } fn update_transfers_meta_by_token_addr_id_sql(chain: &Chain) -> MmResult { - let table_name = nft_transfer_history_table_name(chain); - - validate_table_name(&table_name)?; + let table_name = chain.to_transfer_history_table_name()?; let sql = format!( "UPDATE {} SET token_uri = ?1, token_domain = ?2, collection_name = ?3, image_url = ?4, image_domain = ?5, \ token_name = ?6 WHERE token_address = ?7 AND token_id = ?8;", @@ -466,13 +467,7 @@ fn update_transfers_meta_by_token_addr_id_sql(chain: &Chain) -> MmResult(chain: &Chain, table_name_creator: F) -> MmResult -where - F: FnOnce(&Chain) -> String, -{ - let table_name = table_name_creator(chain); - - validate_table_name(&table_name)?; +fn update_nft_amount_sql(table_name: String) -> MmResult { let sql = format!( "UPDATE {} SET amount = ?1 WHERE token_address = ?2 AND token_id = ?3;", table_name @@ -480,13 +475,7 @@ where Ok(sql) } -fn update_nft_amount_and_block_number_sql(chain: &Chain, table_name_creator: F) -> MmResult -where - F: FnOnce(&Chain) -> String, -{ - let table_name = table_name_creator(chain); - - validate_table_name(&table_name)?; +fn update_nft_amount_and_block_number_sql(table_name: String) -> MmResult { let sql = format!( "UPDATE {} SET amount = ?1, block_number = ?2 WHERE token_address = ?3 AND token_id = ?4;", table_name @@ -495,18 +484,12 @@ where } fn get_nft_metadata_sql(chain: &Chain) -> MmResult { - let table_name = nft_list_table_name(chain); - validate_table_name(&table_name)?; + let table_name = chain.to_nft_list_table_name()?; let sql = format!("SELECT * FROM {} WHERE token_address=?1 AND token_id=?2", table_name); Ok(sql) } -fn select_last_block_number_sql(chain: &Chain, table_name_creator: F) -> MmResult -where - F: FnOnce(&Chain) -> String, -{ - let table_name = table_name_creator(chain); - validate_table_name(&table_name)?; +fn select_last_block_number_sql(table_name: String) -> MmResult { let sql = format!( "SELECT block_number FROM {} ORDER BY block_number DESC LIMIT 1", table_name @@ -515,18 +498,12 @@ where } fn select_last_scanned_block_sql() -> MmResult { - let table_name = scanned_nft_blocks_table_name(); - validate_table_name(&table_name)?; + let table_name = scanned_nft_blocks_table_name()?; let sql = format!("SELECT last_scanned_block FROM {} WHERE chain=?1", table_name,); Ok(sql) } -fn get_nft_amount_sql(chain: &Chain, table_name_creator: F) -> MmResult -where - F: FnOnce(&Chain) -> String, -{ - let table_name = table_name_creator(chain); - validate_table_name(&table_name)?; +fn get_nft_amount_sql(table_name: String) -> MmResult { let sql = format!( "SELECT amount FROM {} WHERE token_address=?1 AND token_id=?2", table_name @@ -534,12 +511,7 @@ where Ok(sql) } -fn delete_nft_sql(chain: &Chain, table_name_creator: F) -> Result> -where - F: FnOnce(&Chain) -> String, -{ - let table_name = table_name_creator(chain); - validate_table_name(&table_name)?; +fn delete_nft_sql(table_name: String) -> Result> { let sql = format!("DELETE FROM {} WHERE token_address=?1 AND token_id=?2", table_name); Ok(sql) } @@ -548,56 +520,20 @@ fn block_number_from_row(row: &Row<'_>) -> Result { row.get::<_, fn nft_amount_from_row(row: &Row<'_>) -> Result { row.get(0) } -fn get_nfts_by_token_address_statement<'a, F>( - conn: &'a Connection, - chain: &'a Chain, - table_name_creator: F, -) -> MmResult, SqlError> -where - F: FnOnce(&Chain) -> String, -{ - let table_name = table_name_creator(chain); - validate_table_name(table_name.as_str())?; +fn get_nfts_by_token_address_statement(conn: &Connection, table_name: String) -> MmResult { let sql_query = format!("SELECT * FROM {} WHERE token_address = ?", table_name); let stmt = conn.prepare(&sql_query)?; Ok(stmt) } -fn get_token_addresses_statement<'a, F>( - conn: &'a Connection, - chain: &'a Chain, - table_name_creator: F, -) -> MmResult, SqlError> -where - F: FnOnce(&Chain) -> String, -{ - let table_name = table_name_creator(chain); - validate_table_name(table_name.as_str())?; +fn get_token_addresses_statement(conn: &Connection, table_name: String) -> MmResult { let sql_query = format!("SELECT DISTINCT token_address FROM {}", table_name); let stmt = conn.prepare(&sql_query)?; Ok(stmt) } -// fn get_transfers_from_block_builder<'a>( -// conn: &'a Connection, -// chain: &'a Chain, -// from_block: u64, -// ) -> MmResult, SqlError> { -// let table_name = nft_transfer_history_table_name(chain); -// validate_table_name(table_name.as_str())?; -// let mut sql_builder = SqlQuery::select_from(conn, table_name.as_str())?; -// sql_builder -// .sql_builder() -// .and_where(format!("block_number >= '{}'", from_block)) -// .order_asc("block_number") -// .field("*"); -// drop_mutability!(sql_builder); -// Ok(sql_builder) -// } - fn get_transfers_from_block_statement<'a>(conn: &'a Connection, chain: &'a Chain) -> MmResult, SqlError> { - let table_name = nft_transfer_history_table_name(chain); - validate_table_name(table_name.as_str())?; + let table_name = chain.to_transfer_history_table_name()?; let sql_query = format!( "SELECT * FROM {} WHERE block_number >= ? ORDER BY block_number ASC", table_name @@ -610,8 +546,7 @@ fn get_transfers_by_token_addr_id_statement<'a>( conn: &'a Connection, chain: &'a Chain, ) -> MmResult, SqlError> { - let table_name = nft_transfer_history_table_name(chain); - validate_table_name(table_name.as_str())?; + let table_name = chain.to_transfer_history_table_name()?; let sql_query = format!("SELECT * FROM {} WHERE token_address = ? AND token_id = ?", table_name); let stmt = conn.prepare(&sql_query)?; Ok(stmt) @@ -621,8 +556,7 @@ fn get_transfers_with_empty_meta_builder<'a>( conn: &'a Connection, chain: &'a Chain, ) -> MmResult, SqlError> { - let table_name = nft_transfer_history_table_name(chain); - validate_table_name(table_name.as_str())?; + let table_name = chain.to_transfer_history_table_name()?; let mut sql_builder = SqlQuery::select_from(conn, table_name.as_str())?; sql_builder .sql_builder() @@ -638,8 +572,7 @@ fn get_transfers_with_empty_meta_builder<'a>( } fn get_transfer_by_tx_hash_and_log_index_sql(chain: &Chain) -> MmResult { - let table_name = nft_transfer_history_table_name(chain); - validate_table_name(&table_name)?; + let table_name = chain.to_transfer_history_table_name()?; let sql = format!( "SELECT * FROM {} WHERE transaction_hash=?1 AND log_index = ?2", table_name @@ -664,8 +597,7 @@ impl NftListStorageOps for SqliteNftStorage { } async fn is_initialized(&self, chain: &Chain) -> MmResult { - let table_name = nft_list_table_name(chain); - validate_table_name(&table_name)?; + let table_name = chain.to_nft_list_table_name()?; let selfi = self.clone(); async_blocking(move || { let conn = selfi.0.lock().unwrap(); @@ -673,7 +605,7 @@ impl NftListStorageOps for SqliteNftStorage { let scanned_nft_blocks_initialized = query_single_row( &conn, CHECK_TABLE_EXISTS_SQL, - [scanned_nft_blocks_table_name()], + [scanned_nft_blocks_table_name()?], string_from_row, )?; Ok(nft_list_initialized.is_some() && scanned_nft_blocks_initialized.is_some()) @@ -800,7 +732,8 @@ impl NftListStorageOps for SqliteNftStorage { token_id: BigDecimal, scanned_block: u64, ) -> MmResult { - let sql = delete_nft_sql(chain, nft_list_table_name)?; + let table_name = chain.to_nft_list_table_name()?; + let sql = delete_nft_sql(table_name)?; let params = [token_address, token_id.to_string()]; let scanned_block_params = [chain.to_ticker(), scanned_block.to_string()]; let selfi = self.clone(); @@ -827,7 +760,8 @@ impl NftListStorageOps for SqliteNftStorage { token_address: String, token_id: BigDecimal, ) -> MmResult, Self::Error> { - let sql = get_nft_amount_sql(chain, nft_list_table_name)?; + let table_name = chain.to_nft_list_table_name()?; + let sql = get_nft_amount_sql(table_name)?; let params = [token_address, token_id.to_string()]; let selfi = self.clone(); async_blocking(move || { @@ -875,7 +809,8 @@ impl NftListStorageOps for SqliteNftStorage { } async fn get_last_block_number(&self, chain: &Chain) -> MmResult, Self::Error> { - let sql = select_last_block_number_sql(chain, nft_list_table_name)?; + let table_name = chain.to_nft_list_table_name()?; + let sql = select_last_block_number_sql(table_name)?; let selfi = self.clone(); async_blocking(move || { let conn = selfi.0.lock().unwrap(); @@ -902,7 +837,8 @@ impl NftListStorageOps for SqliteNftStorage { } async fn update_nft_amount(&self, chain: &Chain, nft: Nft, scanned_block: u64) -> MmResult<(), Self::Error> { - let sql = update_nft_amount_sql(chain, nft_list_table_name)?; + let table_name = chain.to_nft_list_table_name()?; + let sql = update_nft_amount_sql(table_name)?; let scanned_block_params = [chain.to_ticker(), scanned_block.to_string()]; let selfi = self.clone(); async_blocking(move || { @@ -922,7 +858,8 @@ impl NftListStorageOps for SqliteNftStorage { } async fn update_nft_amount_and_block_number(&self, chain: &Chain, nft: Nft) -> MmResult<(), Self::Error> { - let sql = update_nft_amount_and_block_number_sql(chain, nft_list_table_name)?; + let table_name = chain.to_nft_list_table_name()?; + let sql = update_nft_amount_and_block_number_sql(table_name)?; let scanned_block_params = [chain.to_ticker(), nft.block_number.to_string()]; let selfi = self.clone(); async_blocking(move || { @@ -947,7 +884,8 @@ impl NftListStorageOps for SqliteNftStorage { let chain = *chain; async_blocking(move || { let conn = selfi.0.lock().unwrap(); - let mut stmt = get_nfts_by_token_address_statement(&conn, &chain, nft_list_table_name)?; + let table_name = chain.to_nft_list_table_name()?; + let mut stmt = get_nfts_by_token_address_statement(&conn, table_name)?; let nfts = stmt .query_map([token_address], nft_from_row)? .collect::, _>>()?; @@ -964,8 +902,7 @@ impl NftListStorageOps for SqliteNftStorage { ) -> MmResult<(), Self::Error> { let selfi = self.clone(); - let table_name = nft_list_table_name(chain); - validate_table_name(&table_name)?; + let table_name = chain.to_nft_list_table_name()?; let sql = format!("UPDATE {} SET possible_spam = ?1 WHERE token_address = ?2;", table_name); async_blocking(move || { @@ -996,8 +933,7 @@ impl NftTransferHistoryStorageOps for SqliteNftStorage { } async fn is_initialized(&self, chain: &Chain) -> MmResult { - let table_name = nft_transfer_history_table_name(chain); - validate_table_name(&table_name)?; + let table_name = chain.to_transfer_history_table_name()?; let selfi = self.clone(); async_blocking(move || { let conn = selfi.0.lock().unwrap(); @@ -1098,7 +1034,8 @@ impl NftTransferHistoryStorageOps for SqliteNftStorage { } async fn get_last_block_number(&self, chain: &Chain) -> MmResult, Self::Error> { - let sql = select_last_block_number_sql(chain, nft_transfer_history_table_name)?; + let table_name = chain.to_transfer_history_table_name()?; + let sql = select_last_block_number_sql(table_name)?; let selfi = self.clone(); async_blocking(move || { let conn = selfi.0.lock().unwrap(); @@ -1195,9 +1132,8 @@ impl NftTransferHistoryStorageOps for SqliteNftStorage { .await } - async fn get_transfers_with_empty_meta(&self, chain: &Chain) -> MmResult, Self::Error> { + async fn get_transfers_with_empty_meta(&self, chain: Chain) -> MmResult, Self::Error> { let selfi = self.clone(); - let chain = *chain; async_blocking(move || { let conn = selfi.0.lock().unwrap(); let sql_builder = get_transfers_with_empty_meta_builder(&conn, &chain)?; @@ -1216,7 +1152,8 @@ impl NftTransferHistoryStorageOps for SqliteNftStorage { let chain = *chain; async_blocking(move || { let conn = selfi.0.lock().unwrap(); - let mut stmt = get_nfts_by_token_address_statement(&conn, &chain, nft_transfer_history_table_name)?; + let table_name = chain.to_transfer_history_table_name()?; + let mut stmt = get_nfts_by_token_address_statement(&conn, table_name)?; let nfts = stmt .query_map([token_address], transfer_history_from_row)? .collect::, _>>()?; @@ -1233,8 +1170,7 @@ impl NftTransferHistoryStorageOps for SqliteNftStorage { ) -> MmResult<(), Self::Error> { let selfi = self.clone(); - let table_name = nft_transfer_history_table_name(chain); - validate_table_name(&table_name)?; + let table_name = chain.to_transfer_history_table_name()?; let sql = format!("UPDATE {} SET possible_spam = ?1 WHERE token_address = ?2;", table_name); async_blocking(move || { @@ -1248,13 +1184,16 @@ impl NftTransferHistoryStorageOps for SqliteNftStorage { .await } - async fn get_token_addresses(&self, chain: &Chain) -> MmResult, Self::Error> { + async fn get_token_addresses(&self, chain: &Chain) -> MmResult, Self::Error> { let selfi = self.clone(); let chain = *chain; async_blocking(move || { let conn = selfi.0.lock().unwrap(); - let mut stmt = get_token_addresses_statement(&conn, &chain, nft_transfer_history_table_name)?; - let addresses = stmt.query_map([], address_from_row)?.collect::, _>>()?; + let table_name = chain.to_transfer_history_table_name()?; + let mut stmt = get_token_addresses_statement(&conn, table_name)?; + let addresses = stmt + .query_map([], address_from_row)? + .collect::, _>>()?; Ok(addresses) }) .await diff --git a/mm2src/coins/nft/storage/wasm/wasm_storage.rs b/mm2src/coins/nft/storage/wasm/wasm_storage.rs index 2596ec2f6d..f5f280f7e9 100644 --- a/mm2src/coins/nft/storage/wasm/wasm_storage.rs +++ b/mm2src/coins/nft/storage/wasm/wasm_storage.rs @@ -53,7 +53,7 @@ impl IndexedDbNftStorage { }) } - fn take_nfts_according_to_filters(nfts: I, filters: Option) -> WasmNftCacheResult> + fn filter_nfts(nfts: I, filters: Option) -> WasmNftCacheResult> where I: Iterator, { @@ -61,7 +61,7 @@ impl IndexedDbNftStorage { for nft_table in nfts { let nft = nft_details_from_item(nft_table)?; if let Some(filters) = &filters { - if filters.is_spam_match(&nft) && filters.is_phishing_match(&nft) { + if filters.passes_spam_filter(&nft) && filters.passes_phishing_filter(&nft) { filtered_nfts.push(nft); } } else { @@ -87,7 +87,7 @@ impl IndexedDbNftStorage { }) } - fn take_transfers_according_to_filters( + fn filter_transfers( transfers: I, filters: Option, ) -> WasmNftCacheResult> @@ -100,8 +100,8 @@ impl IndexedDbNftStorage { if let Some(filters) = &filters { if filters.is_status_match(&transfer) && filters.is_date_match(&transfer) - && filters.is_spam_match(&transfer) - && filters.is_phishing_match(&transfer) + && filters.passes_spam_filter(&transfer) + && filters.passes_phishing_filter(&transfer) { filtered_transfers.push(transfer); } @@ -114,9 +114,9 @@ impl IndexedDbNftStorage { } impl NftListFilters { - fn is_spam_match(&self, nft: &Nft) -> bool { !self.exclude_spam || !nft.common.possible_spam } + fn passes_spam_filter(&self, nft: &Nft) -> bool { !self.exclude_spam || !nft.common.possible_spam } - fn is_phishing_match(&self, nft: &Nft) -> bool { !self.exclude_phishing || !nft.possible_phishing } + fn passes_phishing_filter(&self, nft: &Nft) -> bool { !self.exclude_phishing || !nft.possible_phishing } } impl NftTransferHistoryFilters { @@ -131,11 +131,11 @@ impl NftTransferHistoryFilters { && self.to_date.map_or(true, |to| transfer.block_timestamp <= to) } - fn is_spam_match(&self, transfer: &NftTransferHistory) -> bool { + fn passes_spam_filter(&self, transfer: &NftTransferHistory) -> bool { !self.exclude_spam || !transfer.common.possible_spam } - fn is_phishing_match(&self, transfer: &NftTransferHistory) -> bool { + fn passes_phishing_filter(&self, transfer: &NftTransferHistory) -> bool { !self.exclude_phishing || !transfer.possible_phishing } } @@ -166,7 +166,7 @@ impl NftListStorageOps for IndexedDbNftStorage { .await? .into_iter() .map(|(_item_id, nft)| nft); - let filtered = Self::take_nfts_according_to_filters(nft_tables, filters)?; + let filtered = Self::filter_nfts(nft_tables, filters)?; nfts.extend(filtered); } Self::take_nft_according_to_paging_opts(nfts, max, limit, page_number) @@ -379,8 +379,8 @@ impl NftListStorageOps for IndexedDbNftStorage { token_address: String, possible_spam: bool, ) -> MmResult<(), Self::Error> { - let nfts: Vec = self.get_nfts_by_token_address(chain, token_address.clone()).await?; let locked_db = self.lock_db().await?; + let nfts: Vec = self.get_nfts_by_token_address(chain, token_address.clone()).await?; let db_transaction = locked_db.get_inner().transaction().await?; let table = db_transaction.table::().await?; @@ -426,7 +426,7 @@ impl NftTransferHistoryStorageOps for IndexedDbNftStorage { .await? .into_iter() .map(|(_item_id, transfer)| transfer); - let filtered = Self::take_transfers_according_to_filters(transfer_tables, filters)?; + let filtered = Self::filter_transfers(transfer_tables, filters)?; transfers.extend(filtered); } Self::take_transfers_according_to_paging_opts(transfers, max, limit, page_number) @@ -531,10 +531,10 @@ impl NftTransferHistoryStorageOps for IndexedDbNftStorage { chain: &Chain, transfer_meta: TransferMeta, ) -> MmResult<(), Self::Error> { + let locked_db = self.lock_db().await?; let transfers: Vec = self .get_transfers_by_token_addr_id(chain, transfer_meta.token_address, transfer_meta.token_id) .await?; - let locked_db = self.lock_db().await?; let db_transaction = locked_db.get_inner().transaction().await?; let table = db_transaction.table::().await?; for mut transfer in transfers { @@ -557,7 +557,7 @@ impl NftTransferHistoryStorageOps for IndexedDbNftStorage { Ok(()) } - async fn get_transfers_with_empty_meta(&self, chain: &Chain) -> MmResult, Self::Error> { + async fn get_transfers_with_empty_meta(&self, chain: Chain) -> MmResult, Self::Error> { let locked_db = self.lock_db().await?; let db_transaction = locked_db.get_inner().transaction().await?; let table = db_transaction.table::().await?; @@ -615,10 +615,10 @@ impl NftTransferHistoryStorageOps for IndexedDbNftStorage { token_address: String, possible_spam: bool, ) -> MmResult<(), Self::Error> { + let locked_db = self.lock_db().await?; let transfers: Vec = self .get_transfers_by_token_address(chain, token_address.clone()) .await?; - let locked_db = self.lock_db().await?; let db_transaction = locked_db.get_inner().transaction().await?; let table = db_transaction.table::().await?; @@ -637,7 +637,7 @@ impl NftTransferHistoryStorageOps for IndexedDbNftStorage { Ok(()) } - async fn get_token_addresses(&self, chain: &Chain) -> MmResult, Self::Error> { + async fn get_token_addresses(&self, chain: &Chain) -> MmResult, Self::Error> { let locked_db = self.lock_db().await?; let db_transaction = locked_db.get_inner().transaction().await?; let table = db_transaction.table::().await?; @@ -648,7 +648,7 @@ impl NftTransferHistoryStorageOps for IndexedDbNftStorage { let transfer = transfer_details_from_item(item)?; token_addresses.insert(transfer.common.token_address); } - Ok(token_addresses.into_iter().collect()) + Ok(token_addresses) } } From 3ae16b83f6dc68a91580cf5d7d2fcd9a71c739f1 Mon Sep 17 00:00:00 2001 From: laruh Date: Wed, 6 Sep 2023 19:30:27 +0700 Subject: [PATCH 11/39] remove Chain ref in some db methods --- mm2src/coins/nft.rs | 58 +++---- mm2src/coins/nft/storage/db_test_helpers.rs | 44 +++--- mm2src/coins/nft/storage/mod.rs | 14 +- mm2src/coins/nft/storage/sql_storage.rs | 145 +++++++----------- mm2src/coins/nft/storage/wasm/wasm_storage.rs | 20 +-- 5 files changed, 122 insertions(+), 159 deletions(-) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index bdb3df043c..d1595a40c2 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -155,48 +155,48 @@ pub async fn update_nft(ctx: MmArc, req: UpdateNftReq) -> MmResult<(), UpdateNft let _lock = nft_ctx.guard.lock().await; let storage = NftStorageBuilder::new(&ctx).build()?; - for chain in req.chains.iter() { - let transfer_history_initialized = NftTransferHistoryStorageOps::is_initialized(&storage, chain).await?; + for chain in req.chains.into_iter() { + let transfer_history_initialized = NftTransferHistoryStorageOps::is_initialized(&storage, &chain).await?; let from_block = if transfer_history_initialized { - let last_transfer_block = NftTransferHistoryStorageOps::get_last_block_number(&storage, chain).await?; + let last_transfer_block = NftTransferHistoryStorageOps::get_last_block_number(&storage, &chain).await?; last_transfer_block.map(|b| b + 1) } else { - NftTransferHistoryStorageOps::init(&storage, chain).await?; + NftTransferHistoryStorageOps::init(&storage, &chain).await?; None }; - let nft_transfers = get_moralis_nft_transfers(&ctx, chain, from_block, &req.url).await?; + let nft_transfers = get_moralis_nft_transfers(&ctx, &chain, from_block, &req.url).await?; storage.add_transfers_to_history(chain, nft_transfers).await?; - let nft_block = match NftListStorageOps::get_last_block_number(&storage, chain).await { + let nft_block = match NftListStorageOps::get_last_block_number(&storage, &chain).await { Ok(Some(block)) => block, Ok(None) => { // if there are no rows in NFT LIST table we can try to get all info from moralis. - let nfts = cache_nfts_from_moralis(&ctx, &storage, chain, &req.url).await?; - update_meta_in_transfers(&storage, chain, nfts).await?; - update_transfers_with_empty_meta(&storage, chain, &req.url).await?; - update_spam(&ctx, &storage, *chain, &req.url_antispam).await?; + let nfts = cache_nfts_from_moralis(&ctx, &storage, &chain, &req.url).await?; + update_meta_in_transfers(&storage, &chain, nfts).await?; + update_transfers_with_empty_meta(&storage, &chain, &req.url).await?; + update_spam(&ctx, &storage, chain, &req.url_antispam).await?; continue; }, Err(_) => { // if there is an error, then NFT LIST table doesnt exist, so we need to cache from mroalis. - NftListStorageOps::init(&storage, chain).await?; - let nft_list = get_moralis_nft_list(&ctx, chain, &req.url).await?; - let last_scanned_block = NftTransferHistoryStorageOps::get_last_block_number(&storage, chain) + NftListStorageOps::init(&storage, &chain).await?; + let nft_list = get_moralis_nft_list(&ctx, &chain, &req.url).await?; + let last_scanned_block = NftTransferHistoryStorageOps::get_last_block_number(&storage, &chain) .await? .unwrap_or(0); storage .add_nfts_to_list(chain, nft_list.clone(), last_scanned_block) .await?; - update_meta_in_transfers(&storage, chain, nft_list).await?; - update_transfers_with_empty_meta(&storage, chain, &req.url).await?; - update_spam(&ctx, &storage, *chain, &req.url_antispam).await?; + update_meta_in_transfers(&storage, &chain, nft_list).await?; + update_transfers_with_empty_meta(&storage, &chain, &req.url).await?; + update_spam(&ctx, &storage, chain, &req.url_antispam).await?; continue; }, }; let scanned_block = storage - .get_last_scanned_block(chain) + .get_last_scanned_block(&chain) .await? .ok_or_else(|| UpdateNftError::LastScannedBlockNotFound { last_nft_block: nft_block.to_string(), @@ -209,9 +209,9 @@ pub async fn update_nft(ctx: MmArc, req: UpdateNftReq) -> MmResult<(), UpdateNft last_nft_block: nft_block.to_string(), }); } - update_nft_list(ctx.clone(), &storage, chain, scanned_block + 1, &req.url).await?; - update_transfers_with_empty_meta(&storage, chain, &req.url).await?; - update_spam(&ctx, &storage, *chain, &req.url_antispam).await?; + update_nft_list(ctx.clone(), &storage, &chain, scanned_block + 1, &req.url).await?; + update_transfers_with_empty_meta(&storage, &chain, &req.url).await?; + update_spam(&ctx, &storage, chain, &req.url_antispam).await?; // todo update phishing domains } Ok(()) @@ -230,7 +230,7 @@ where if chain == Chain::Eth || chain == Chain::Polygon { return update_spam_nft_with_mnemonichq(ctx, storage, &chain, url_antispam).await; } - let token_addresses = storage.get_token_addresses(&chain).await?; + let token_addresses = storage.get_token_addresses(chain).await?; let addresses = token_addresses .iter() .map(eth_addr_to_hex) @@ -422,7 +422,7 @@ async fn get_moralis_nft_list(ctx: &MmArc, chain: &Chain, url: &Url) -> MmResult Some(contract_type) => contract_type, None => continue, }; - let nft = build_nft_from_moralis(chain, nft_moralis, contract_type).await; + let nft = build_nft_from_moralis(*chain, nft_moralis, contract_type).await; // collect NFTs from the page res_list.push(nft); } @@ -560,7 +560,7 @@ async fn get_moralis_metadata( Some(contract_type) => contract_type, None => return MmError::err(GetNftInfoError::ContractTypeIsNull), }; - let nft_metadata = build_nft_from_moralis(chain, nft_moralis, contract_type).await; + let nft_metadata = build_nft_from_moralis(*chain, nft_moralis, contract_type).await; Ok(nft_metadata) } @@ -700,7 +700,7 @@ async fn update_nft_list( scan_from_block: u64, url: &Url, ) -> MmResult<(), UpdateNftError> { - let transfers = storage.get_transfers_from_block(chain, scan_from_block).await?; + let transfers = storage.get_transfers_from_block(*chain, scan_from_block).await?; let req = MyAddressReq { coin: chain.to_ticker(), path_to_address: StandardHDCoinAddress::default(), @@ -808,7 +808,7 @@ async fn handle_receive_erc721 Nft { +async fn build_nft_from_moralis(chain: Chain, nft_moralis: NftFromMoralis, contract_type: ContractType) -> Nft { let token_uri = check_moralis_ipfs_bafy(nft_moralis.common.token_uri.as_deref()); let uri_meta = get_uri_meta(token_uri.as_deref(), nft_moralis.common.metadata.as_deref()).await; let token_domain = get_domain_from_url(token_uri.as_deref()); @@ -1130,7 +1130,7 @@ async fn build_nft_from_moralis(chain: &Chain, nft_moralis: NftFromMoralis, cont minter_address: nft_moralis.common.minter_address, possible_spam: nft_moralis.common.possible_spam, }, - chain: *chain, + chain, block_number_minted: nft_moralis.block_number_minted.map(|v| v.0), block_number: *nft_moralis.block_number, contract_type, diff --git a/mm2src/coins/nft/storage/db_test_helpers.rs b/mm2src/coins/nft/storage/db_test_helpers.rs index 0a0a388b2c..d2e27d748b 100644 --- a/mm2src/coins/nft/storage/db_test_helpers.rs +++ b/mm2src/coins/nft/storage/db_test_helpers.rs @@ -379,7 +379,7 @@ pub(crate) async fn test_add_get_nfts_impl() { let chain = Chain::Bsc; let storage = init_nft_list_storage(&chain).await; let nft_list = nft_list(); - storage.add_nfts_to_list(&chain, nft_list, 28056726).await.unwrap(); + storage.add_nfts_to_list(chain, nft_list, 28056726).await.unwrap(); let token_id = BigDecimal::from_str(TOKEN_ID).unwrap(); let nft = storage @@ -394,7 +394,7 @@ pub(crate) async fn test_last_nft_block_impl() { let chain = Chain::Bsc; let storage = init_nft_list_storage(&chain).await; let nft_list = nft_list(); - storage.add_nfts_to_list(&chain, nft_list, 28056726).await.unwrap(); + storage.add_nfts_to_list(chain, nft_list, 28056726).await.unwrap(); let last_block = NftListStorageOps::get_last_block_number(&storage, &chain) .await @@ -407,7 +407,7 @@ pub(crate) async fn test_nft_list_impl() { let chain = Chain::Bsc; let storage = init_nft_list_storage(&chain).await; let nft_list = nft_list(); - storage.add_nfts_to_list(&chain, nft_list, 28056726).await.unwrap(); + storage.add_nfts_to_list(chain, nft_list, 28056726).await.unwrap(); let nft_list = storage .get_nft_list(vec![chain], false, 1, Some(NonZeroUsize::new(3).unwrap()), None) @@ -424,7 +424,7 @@ pub(crate) async fn test_remove_nft_impl() { let chain = Chain::Bsc; let storage = init_nft_list_storage(&chain).await; let nft_list = nft_list(); - storage.add_nfts_to_list(&chain, nft_list, 28056726).await.unwrap(); + storage.add_nfts_to_list(chain, nft_list, 28056726).await.unwrap(); let token_id = BigDecimal::from_str(TOKEN_ID).unwrap(); let remove_rslt = storage @@ -448,7 +448,7 @@ pub(crate) async fn test_nft_amount_impl() { let storage = init_nft_list_storage(&chain).await; let mut nft = nft(); storage - .add_nfts_to_list(&chain, vec![nft.clone()], 25919780) + .add_nfts_to_list(chain, vec![nft.clone()], 25919780) .await .unwrap(); @@ -489,7 +489,7 @@ pub(crate) async fn test_refresh_metadata_impl() { let new_symbol = "NEW_SYMBOL"; let mut nft = nft(); storage - .add_nfts_to_list(&chain, vec![nft.clone()], 25919780) + .add_nfts_to_list(chain, vec![nft.clone()], 25919780) .await .unwrap(); nft.common.symbol = Some(new_symbol.to_string()); @@ -505,14 +505,14 @@ pub(crate) async fn test_update_nft_spam_by_token_address_impl() { let chain = Chain::Bsc; let storage = init_nft_list_storage(&chain).await; let nft_list = nft_list(); - storage.add_nfts_to_list(&chain, nft_list, 28056726).await.unwrap(); + storage.add_nfts_to_list(chain, nft_list, 28056726).await.unwrap(); storage .update_nft_spam_by_token_address(&chain, TOKEN_ADD.to_string(), true) .await .unwrap(); let nfts = storage - .get_nfts_by_token_address(&chain, TOKEN_ADD.to_string()) + .get_nfts_by_token_address(chain, TOKEN_ADD.to_string()) .await .unwrap(); for nft in nfts { @@ -524,7 +524,7 @@ pub(crate) async fn test_exclude_nft_spam_impl() { let chain = Chain::Bsc; let storage = init_nft_list_storage(&chain).await; let nft_list = nft_list(); - storage.add_nfts_to_list(&chain, nft_list, 28056726).await.unwrap(); + storage.add_nfts_to_list(chain, nft_list, 28056726).await.unwrap(); let filters = NftListFilters { exclude_spam: true, @@ -541,11 +541,11 @@ pub(crate) async fn test_add_get_transfers_impl() { let chain = Chain::Bsc; let storage = init_nft_history_storage(&chain).await; let transfers = nft_transfer_history(); - storage.add_transfers_to_history(&chain, transfers).await.unwrap(); + storage.add_transfers_to_history(chain, transfers).await.unwrap(); let token_id = BigDecimal::from_str(TOKEN_ID).unwrap(); let transfer1 = storage - .get_transfers_by_token_addr_id(&chain, TOKEN_ADD.to_string(), token_id) + .get_transfers_by_token_addr_id(chain, TOKEN_ADD.to_string(), token_id) .await .unwrap() .get(0) @@ -558,7 +558,7 @@ pub(crate) async fn test_add_get_transfers_impl() { .unwrap() .unwrap(); assert_eq!(transfer2.block_number, 28056726); - let transfer_from = storage.get_transfers_from_block(&chain, 28056721).await.unwrap(); + let transfer_from = storage.get_transfers_from_block(chain, 28056721).await.unwrap(); assert_eq!(transfer_from.len(), 3); } @@ -566,7 +566,7 @@ pub(crate) async fn test_last_transfer_block_impl() { let chain = Chain::Bsc; let storage = init_nft_history_storage(&chain).await; let transfers = nft_transfer_history(); - storage.add_transfers_to_history(&chain, transfers).await.unwrap(); + storage.add_transfers_to_history(chain, transfers).await.unwrap(); let last_block = NftTransferHistoryStorageOps::get_last_block_number(&storage, &chain) .await @@ -579,7 +579,7 @@ pub(crate) async fn test_transfer_history_impl() { let chain = Chain::Bsc; let storage = init_nft_history_storage(&chain).await; let transfers = nft_transfer_history(); - storage.add_transfers_to_history(&chain, transfers).await.unwrap(); + storage.add_transfers_to_history(chain, transfers).await.unwrap(); let transfer_history = storage .get_transfer_history(vec![chain], false, 1, Some(NonZeroUsize::new(3).unwrap()), None) @@ -596,7 +596,7 @@ pub(crate) async fn test_transfer_history_filters_impl() { let chain = Chain::Bsc; let storage = init_nft_history_storage(&chain).await; let transfers = nft_transfer_history(); - storage.add_transfers_to_history(&chain, transfers).await.unwrap(); + storage.add_transfers_to_history(chain, transfers).await.unwrap(); let filters = NftTransferHistoryFilters { receive: true, @@ -656,7 +656,7 @@ pub(crate) async fn test_get_update_transfer_meta_impl() { let chain = Chain::Bsc; let storage = init_nft_history_storage(&chain).await; let transfers = nft_transfer_history(); - storage.add_transfers_to_history(&chain, transfers).await.unwrap(); + storage.add_transfers_to_history(chain, transfers).await.unwrap(); let vec_token_add_id = storage.get_transfers_with_empty_meta(chain).await.unwrap(); assert_eq!(vec_token_add_id.len(), 3); @@ -677,7 +677,7 @@ pub(crate) async fn test_get_update_transfer_meta_impl() { .await .unwrap(); let transfer_upd = storage - .get_transfers_by_token_addr_id(&chain, token_add, Default::default()) + .get_transfers_by_token_addr_id(chain, token_add, Default::default()) .await .unwrap(); let transfer_upd = transfer_upd.get(0).unwrap(); @@ -688,14 +688,14 @@ pub(crate) async fn test_update_transfer_spam_by_token_address_impl() { let chain = Chain::Bsc; let storage = init_nft_history_storage(&chain).await; let transfers = nft_transfer_history(); - storage.add_transfers_to_history(&chain, transfers).await.unwrap(); + storage.add_transfers_to_history(chain, transfers).await.unwrap(); storage .update_transfer_spam_by_token_address(&chain, TOKEN_ADD.to_string(), true) .await .unwrap(); let transfers = storage - .get_transfers_by_token_address(&chain, TOKEN_ADD.to_string()) + .get_transfers_by_token_address(chain, TOKEN_ADD.to_string()) .await .unwrap(); for transfers in transfers { @@ -707,9 +707,9 @@ pub(crate) async fn test_get_token_addresses_impl() { let chain = Chain::Bsc; let storage = init_nft_history_storage(&chain).await; let transfers = nft_transfer_history(); - storage.add_transfers_to_history(&chain, transfers).await.unwrap(); + storage.add_transfers_to_history(chain, transfers).await.unwrap(); - let token_addresses = storage.get_token_addresses(&chain).await.unwrap(); + let token_addresses = storage.get_token_addresses(chain).await.unwrap(); assert_eq!(token_addresses.len(), 2); } @@ -717,7 +717,7 @@ pub(crate) async fn test_exclude_transfer_spam_impl() { let chain = Chain::Bsc; let storage = init_nft_history_storage(&chain).await; let transfers = nft_transfer_history(); - storage.add_transfers_to_history(&chain, transfers).await.unwrap(); + storage.add_transfers_to_history(chain, transfers).await.unwrap(); let filters = NftTransferHistoryFilters { receive: true, diff --git a/mm2src/coins/nft/storage/mod.rs b/mm2src/coins/nft/storage/mod.rs index a9063171a2..dea9964cfc 100644 --- a/mm2src/coins/nft/storage/mod.rs +++ b/mm2src/coins/nft/storage/mod.rs @@ -48,7 +48,7 @@ pub trait NftListStorageOps { filters: Option, ) -> MmResult; - async fn add_nfts_to_list(&self, chain: &Chain, nfts: I, last_scanned_block: u64) -> MmResult<(), Self::Error> + async fn add_nfts_to_list(&self, chain: Chain, nfts: I, last_scanned_block: u64) -> MmResult<(), Self::Error> where I: IntoIterator + Send + 'static, I::IntoIter: Send; @@ -90,7 +90,7 @@ pub trait NftListStorageOps { async fn update_nft_amount_and_block_number(&self, chain: &Chain, nft: Nft) -> MmResult<(), Self::Error>; /// `get_nfts_by_token_address` function returns list of NFTs which have specified token address. - async fn get_nfts_by_token_address(&self, chain: &Chain, token_address: String) -> MmResult, Self::Error>; + async fn get_nfts_by_token_address(&self, chain: Chain, token_address: String) -> MmResult, Self::Error>; /// `update_nft_spam_by_token_address` function updates `possible_spam` field in NFTs which have specified token address. async fn update_nft_spam_by_token_address( @@ -120,7 +120,7 @@ pub trait NftTransferHistoryStorageOps { filters: Option, ) -> MmResult; - async fn add_transfers_to_history(&self, chain: &Chain, transfers: I) -> MmResult<(), Self::Error> + async fn add_transfers_to_history(&self, chain: Chain, transfers: I) -> MmResult<(), Self::Error> where I: IntoIterator + Send + 'static, I::IntoIter: Send; @@ -131,13 +131,13 @@ pub trait NftTransferHistoryStorageOps { /// block_number in ascending order. It is needed to update the NFT LIST table correctly. async fn get_transfers_from_block( &self, - chain: &Chain, + chain: Chain, from_block: u64, ) -> MmResult, Self::Error>; async fn get_transfers_by_token_addr_id( &self, - chain: &Chain, + chain: Chain, token_address: String, token_id: BigDecimal, ) -> MmResult, Self::Error>; @@ -160,7 +160,7 @@ pub trait NftTransferHistoryStorageOps { /// `get_transfers_by_token_address` function returns list of NFT transfers which have specified token address. async fn get_transfers_by_token_address( &self, - chain: &Chain, + chain: Chain, token_address: String, ) -> MmResult, Self::Error>; @@ -173,7 +173,7 @@ pub trait NftTransferHistoryStorageOps { ) -> MmResult<(), Self::Error>; /// `get_token_addresses` return all unique token addresses. - async fn get_token_addresses(&self, chain: &Chain) -> MmResult, Self::Error>; + async fn get_token_addresses(&self, chain: Chain) -> MmResult, Self::Error>; } #[derive(Debug, Deserialize, Display, Serialize)] diff --git a/mm2src/coins/nft/storage/sql_storage.rs b/mm2src/coins/nft/storage/sql_storage.rs index 976972b26b..ef0bb30165 100644 --- a/mm2src/coins/nft/storage/sql_storage.rs +++ b/mm2src/coins/nft/storage/sql_storage.rs @@ -25,13 +25,13 @@ use std::str::FromStr; use std::sync::{Arc, Mutex}; impl Chain { - fn to_nft_list_table_name(self) -> SqlResult { + fn nft_list_table_name(&self) -> SqlResult { let name = self.to_ticker() + "_nft_list"; validate_table_name(&name)?; Ok(name) } - fn to_transfer_history_table_name(self) -> SqlResult { + fn transfer_history_table_name(&self) -> SqlResult { let name = self.to_ticker() + "_nft_transfer_history"; validate_table_name(&name)?; Ok(name) @@ -45,7 +45,7 @@ fn scanned_nft_blocks_table_name() -> SqlResult { } fn create_nft_list_table_sql(chain: &Chain) -> MmResult { - let table_name = chain.to_nft_list_table_name()?; + let table_name = chain.nft_list_table_name()?; let sql = format!( "CREATE TABLE IF NOT EXISTS {} ( token_address VARCHAR(256) NOT NULL, @@ -83,7 +83,7 @@ fn create_nft_list_table_sql(chain: &Chain) -> MmResult { } fn create_transfer_history_table_sql(chain: &Chain) -> MmResult { - let table_name = chain.to_transfer_history_table_name()?; + let table_name = chain.transfer_history_table_name()?; let sql = format!( "CREATE TABLE IF NOT EXISTS {} ( transaction_hash VARCHAR(256) NOT NULL, @@ -147,7 +147,7 @@ fn get_nft_list_builder_preimage( let union_sql_strings = chains .iter() .map(|chain| { - let table_name = chain.to_nft_list_table_name()?; + let table_name = chain.nft_list_table_name()?; let sql_builder = nft_list_builder_preimage(table_name.as_str(), filters)?; let sql_string = sql_builder .sql() @@ -185,7 +185,7 @@ fn get_nft_transfer_builder_preimage( let union_sql_strings = chains .into_iter() .map(|chain| { - let table_name = chain.to_transfer_history_table_name()?; + let table_name = chain.transfer_history_table_name()?; let sql_builder = nft_history_table_builder_preimage(table_name.as_str(), filters)?; let sql_string = sql_builder .sql() @@ -405,7 +405,7 @@ fn token_address_id_from_row(row: &Row<'_>) -> Result } fn insert_nft_in_list_sql(chain: &Chain) -> MmResult { - let table_name = chain.to_nft_list_table_name()?; + let table_name = chain.nft_list_table_name()?; let sql = format!( "INSERT INTO {} ( token_address, token_id, chain, amount, block_number, contract_type, possible_spam, @@ -423,7 +423,7 @@ fn insert_nft_in_list_sql(chain: &Chain) -> MmResult { } fn insert_transfer_in_history_sql(chain: &Chain) -> MmResult { - let table_name = chain.to_transfer_history_table_name()?; + let table_name = chain.transfer_history_table_name()?; let sql = format!( "INSERT INTO {} ( transaction_hash, log_index, chain, block_number, block_timestamp, contract_type, @@ -447,7 +447,7 @@ fn upsert_last_scanned_block_sql() -> MmResult { } fn refresh_nft_metadata_sql(chain: &Chain) -> MmResult { - let table_name = chain.to_nft_list_table_name()?; + let table_name = chain.nft_list_table_name()?; let sql = format!( "UPDATE {} SET possible_spam = ?1, possible_phishing = ?2, collection_name = ?3, symbol = ?4, token_uri = ?5, token_domain = ?6, metadata = ?7, \ last_token_uri_sync = ?8, last_metadata_sync = ?9, raw_image_url = ?10, image_url = ?11, image_domain = ?12, token_name = ?13, description = ?14, \ @@ -458,7 +458,7 @@ fn refresh_nft_metadata_sql(chain: &Chain) -> MmResult { } fn update_transfers_meta_by_token_addr_id_sql(chain: &Chain) -> MmResult { - let table_name = chain.to_transfer_history_table_name()?; + let table_name = chain.transfer_history_table_name()?; let sql = format!( "UPDATE {} SET token_uri = ?1, token_domain = ?2, collection_name = ?3, image_url = ?4, image_domain = ?5, \ token_name = ?6 WHERE token_address = ?7 AND token_id = ?8;", @@ -467,28 +467,6 @@ fn update_transfers_meta_by_token_addr_id_sql(chain: &Chain) -> MmResult MmResult { - let sql = format!( - "UPDATE {} SET amount = ?1 WHERE token_address = ?2 AND token_id = ?3;", - table_name - ); - Ok(sql) -} - -fn update_nft_amount_and_block_number_sql(table_name: String) -> MmResult { - let sql = format!( - "UPDATE {} SET amount = ?1, block_number = ?2 WHERE token_address = ?3 AND token_id = ?4;", - table_name - ); - Ok(sql) -} - -fn get_nft_metadata_sql(chain: &Chain) -> MmResult { - let table_name = chain.to_nft_list_table_name()?; - let sql = format!("SELECT * FROM {} WHERE token_address=?1 AND token_id=?2", table_name); - Ok(sql) -} - fn select_last_block_number_sql(table_name: String) -> MmResult { let sql = format!( "SELECT block_number FROM {} ORDER BY block_number DESC LIMIT 1", @@ -503,14 +481,6 @@ fn select_last_scanned_block_sql() -> MmResult { Ok(sql) } -fn get_nft_amount_sql(table_name: String) -> MmResult { - let sql = format!( - "SELECT amount FROM {} WHERE token_address=?1 AND token_id=?2", - table_name - ); - Ok(sql) -} - fn delete_nft_sql(table_name: String) -> Result> { let sql = format!("DELETE FROM {} WHERE token_address=?1 AND token_id=?2", table_name); Ok(sql) @@ -533,7 +503,7 @@ fn get_token_addresses_statement(conn: &Connection, table_name: String) -> MmRes } fn get_transfers_from_block_statement<'a>(conn: &'a Connection, chain: &'a Chain) -> MmResult, SqlError> { - let table_name = chain.to_transfer_history_table_name()?; + let table_name = chain.transfer_history_table_name()?; let sql_query = format!( "SELECT * FROM {} WHERE block_number >= ? ORDER BY block_number ASC", table_name @@ -542,11 +512,8 @@ fn get_transfers_from_block_statement<'a>(conn: &'a Connection, chain: &'a Chain Ok(stmt) } -fn get_transfers_by_token_addr_id_statement<'a>( - conn: &'a Connection, - chain: &'a Chain, -) -> MmResult, SqlError> { - let table_name = chain.to_transfer_history_table_name()?; +fn get_transfers_by_token_addr_id_statement(conn: &Connection, chain: Chain) -> MmResult { + let table_name = chain.transfer_history_table_name()?; let sql_query = format!("SELECT * FROM {} WHERE token_address = ? AND token_id = ?", table_name); let stmt = conn.prepare(&sql_query)?; Ok(stmt) @@ -556,7 +523,7 @@ fn get_transfers_with_empty_meta_builder<'a>( conn: &'a Connection, chain: &'a Chain, ) -> MmResult, SqlError> { - let table_name = chain.to_transfer_history_table_name()?; + let table_name = chain.transfer_history_table_name()?; let mut sql_builder = SqlQuery::select_from(conn, table_name.as_str())?; sql_builder .sql_builder() @@ -571,15 +538,6 @@ fn get_transfers_with_empty_meta_builder<'a>( Ok(sql_builder) } -fn get_transfer_by_tx_hash_and_log_index_sql(chain: &Chain) -> MmResult { - let table_name = chain.to_transfer_history_table_name()?; - let sql = format!( - "SELECT * FROM {} WHERE transaction_hash=?1 AND log_index = ?2", - table_name - ); - Ok(sql) -} - #[async_trait] impl NftListStorageOps for SqliteNftStorage { type Error = SqlError; @@ -597,7 +555,7 @@ impl NftListStorageOps for SqliteNftStorage { } async fn is_initialized(&self, chain: &Chain) -> MmResult { - let table_name = chain.to_nft_list_table_name()?; + let table_name = chain.nft_list_table_name()?; let selfi = self.clone(); async_blocking(move || { let conn = selfi.0.lock().unwrap(); @@ -651,13 +609,12 @@ impl NftListStorageOps for SqliteNftStorage { .await } - async fn add_nfts_to_list(&self, chain: &Chain, nfts: I, last_scanned_block: u64) -> MmResult<(), Self::Error> + async fn add_nfts_to_list(&self, chain: Chain, nfts: I, last_scanned_block: u64) -> MmResult<(), Self::Error> where I: IntoIterator + Send + 'static, I::IntoIter: Send, { let selfi = self.clone(); - let chain = *chain; async_blocking(move || { let mut conn = selfi.0.lock().unwrap(); let sql_transaction = conn.transaction()?; @@ -715,7 +672,8 @@ impl NftListStorageOps for SqliteNftStorage { token_address: String, token_id: BigDecimal, ) -> MmResult, Self::Error> { - let sql = get_nft_metadata_sql(chain)?; + let table_name = chain.nft_list_table_name()?; + let sql = format!("SELECT * FROM {} WHERE token_address=?1 AND token_id=?2", table_name); let params = [token_address, token_id.to_string()]; let selfi = self.clone(); async_blocking(move || { @@ -732,7 +690,7 @@ impl NftListStorageOps for SqliteNftStorage { token_id: BigDecimal, scanned_block: u64, ) -> MmResult { - let table_name = chain.to_nft_list_table_name()?; + let table_name = chain.nft_list_table_name()?; let sql = delete_nft_sql(table_name)?; let params = [token_address, token_id.to_string()]; let scanned_block_params = [chain.to_ticker(), scanned_block.to_string()]; @@ -760,8 +718,11 @@ impl NftListStorageOps for SqliteNftStorage { token_address: String, token_id: BigDecimal, ) -> MmResult, Self::Error> { - let table_name = chain.to_nft_list_table_name()?; - let sql = get_nft_amount_sql(table_name)?; + let table_name = chain.nft_list_table_name()?; + let sql = format!( + "SELECT amount FROM {} WHERE token_address=?1 AND token_id=?2", + table_name + ); let params = [token_address, token_id.to_string()]; let selfi = self.clone(); async_blocking(move || { @@ -809,7 +770,7 @@ impl NftListStorageOps for SqliteNftStorage { } async fn get_last_block_number(&self, chain: &Chain) -> MmResult, Self::Error> { - let table_name = chain.to_nft_list_table_name()?; + let table_name = chain.nft_list_table_name()?; let sql = select_last_block_number_sql(table_name)?; let selfi = self.clone(); async_blocking(move || { @@ -837,8 +798,11 @@ impl NftListStorageOps for SqliteNftStorage { } async fn update_nft_amount(&self, chain: &Chain, nft: Nft, scanned_block: u64) -> MmResult<(), Self::Error> { - let table_name = chain.to_nft_list_table_name()?; - let sql = update_nft_amount_sql(table_name)?; + let table_name = chain.nft_list_table_name()?; + let sql = format!( + "UPDATE {} SET amount = ?1 WHERE token_address = ?2 AND token_id = ?3;", + table_name + ); let scanned_block_params = [chain.to_ticker(), scanned_block.to_string()]; let selfi = self.clone(); async_blocking(move || { @@ -858,8 +822,11 @@ impl NftListStorageOps for SqliteNftStorage { } async fn update_nft_amount_and_block_number(&self, chain: &Chain, nft: Nft) -> MmResult<(), Self::Error> { - let table_name = chain.to_nft_list_table_name()?; - let sql = update_nft_amount_and_block_number_sql(table_name)?; + let table_name = chain.nft_list_table_name()?; + let sql = format!( + "UPDATE {} SET amount = ?1, block_number = ?2 WHERE token_address = ?3 AND token_id = ?4;", + table_name + ); let scanned_block_params = [chain.to_ticker(), nft.block_number.to_string()]; let selfi = self.clone(); async_blocking(move || { @@ -879,12 +846,11 @@ impl NftListStorageOps for SqliteNftStorage { .await } - async fn get_nfts_by_token_address(&self, chain: &Chain, token_address: String) -> MmResult, Self::Error> { + async fn get_nfts_by_token_address(&self, chain: Chain, token_address: String) -> MmResult, Self::Error> { let selfi = self.clone(); - let chain = *chain; async_blocking(move || { let conn = selfi.0.lock().unwrap(); - let table_name = chain.to_nft_list_table_name()?; + let table_name = chain.nft_list_table_name()?; let mut stmt = get_nfts_by_token_address_statement(&conn, table_name)?; let nfts = stmt .query_map([token_address], nft_from_row)? @@ -901,10 +867,8 @@ impl NftListStorageOps for SqliteNftStorage { possible_spam: bool, ) -> MmResult<(), Self::Error> { let selfi = self.clone(); - - let table_name = chain.to_nft_list_table_name()?; + let table_name = chain.nft_list_table_name()?; let sql = format!("UPDATE {} SET possible_spam = ?1 WHERE token_address = ?2;", table_name); - async_blocking(move || { let mut conn = selfi.0.lock().unwrap(); let sql_transaction = conn.transaction()?; @@ -933,7 +897,7 @@ impl NftTransferHistoryStorageOps for SqliteNftStorage { } async fn is_initialized(&self, chain: &Chain) -> MmResult { - let table_name = chain.to_transfer_history_table_name()?; + let table_name = chain.transfer_history_table_name()?; let selfi = self.clone(); async_blocking(move || { let conn = selfi.0.lock().unwrap(); @@ -981,13 +945,12 @@ impl NftTransferHistoryStorageOps for SqliteNftStorage { .await } - async fn add_transfers_to_history(&self, chain: &Chain, transfers: I) -> MmResult<(), Self::Error> + async fn add_transfers_to_history(&self, chain: Chain, transfers: I) -> MmResult<(), Self::Error> where I: IntoIterator + Send + 'static, I::IntoIter: Send, { let selfi = self.clone(); - let chain = *chain; async_blocking(move || { let mut conn = selfi.0.lock().unwrap(); let sql_transaction = conn.transaction()?; @@ -1034,7 +997,7 @@ impl NftTransferHistoryStorageOps for SqliteNftStorage { } async fn get_last_block_number(&self, chain: &Chain) -> MmResult, Self::Error> { - let table_name = chain.to_transfer_history_table_name()?; + let table_name = chain.transfer_history_table_name()?; let sql = select_last_block_number_sql(table_name)?; let selfi = self.clone(); async_blocking(move || { @@ -1049,11 +1012,10 @@ impl NftTransferHistoryStorageOps for SqliteNftStorage { async fn get_transfers_from_block( &self, - chain: &Chain, + chain: Chain, from_block: u64, ) -> MmResult, Self::Error> { let selfi = self.clone(); - let chain = *chain; async_blocking(move || { let conn = selfi.0.lock().unwrap(); let mut stmt = get_transfers_from_block_statement(&conn, &chain)?; @@ -1067,15 +1029,14 @@ impl NftTransferHistoryStorageOps for SqliteNftStorage { async fn get_transfers_by_token_addr_id( &self, - chain: &Chain, + chain: Chain, token_address: String, token_id: BigDecimal, ) -> MmResult, Self::Error> { let selfi = self.clone(); - let chain = *chain; async_blocking(move || { let conn = selfi.0.lock().unwrap(); - let mut stmt = get_transfers_by_token_addr_id_statement(&conn, &chain)?; + let mut stmt = get_transfers_by_token_addr_id_statement(&conn, chain)?; let transfers = stmt .query_map([token_address, token_id.to_string()], transfer_history_from_row)? .collect::, _>>()?; @@ -1090,7 +1051,11 @@ impl NftTransferHistoryStorageOps for SqliteNftStorage { transaction_hash: String, log_index: u32, ) -> MmResult, Self::Error> { - let sql = get_transfer_by_tx_hash_and_log_index_sql(chain)?; + let table_name = chain.transfer_history_table_name()?; + let sql = format!( + "SELECT * FROM {} WHERE transaction_hash=?1 AND log_index = ?2", + table_name + ); let selfi = self.clone(); async_blocking(move || { let conn = selfi.0.lock().unwrap(); @@ -1145,14 +1110,13 @@ impl NftTransferHistoryStorageOps for SqliteNftStorage { async fn get_transfers_by_token_address( &self, - chain: &Chain, + chain: Chain, token_address: String, ) -> MmResult, Self::Error> { let selfi = self.clone(); - let chain = *chain; async_blocking(move || { let conn = selfi.0.lock().unwrap(); - let table_name = chain.to_transfer_history_table_name()?; + let table_name = chain.transfer_history_table_name()?; let mut stmt = get_nfts_by_token_address_statement(&conn, table_name)?; let nfts = stmt .query_map([token_address], transfer_history_from_row)? @@ -1170,7 +1134,7 @@ impl NftTransferHistoryStorageOps for SqliteNftStorage { ) -> MmResult<(), Self::Error> { let selfi = self.clone(); - let table_name = chain.to_transfer_history_table_name()?; + let table_name = chain.transfer_history_table_name()?; let sql = format!("UPDATE {} SET possible_spam = ?1 WHERE token_address = ?2;", table_name); async_blocking(move || { @@ -1184,12 +1148,11 @@ impl NftTransferHistoryStorageOps for SqliteNftStorage { .await } - async fn get_token_addresses(&self, chain: &Chain) -> MmResult, Self::Error> { + async fn get_token_addresses(&self, chain: Chain) -> MmResult, Self::Error> { let selfi = self.clone(); - let chain = *chain; async_blocking(move || { let conn = selfi.0.lock().unwrap(); - let table_name = chain.to_transfer_history_table_name()?; + let table_name = chain.transfer_history_table_name()?; let mut stmt = get_token_addresses_statement(&conn, table_name)?; let addresses = stmt .query_map([], address_from_row)? diff --git a/mm2src/coins/nft/storage/wasm/wasm_storage.rs b/mm2src/coins/nft/storage/wasm/wasm_storage.rs index f5f280f7e9..bde9923ae2 100644 --- a/mm2src/coins/nft/storage/wasm/wasm_storage.rs +++ b/mm2src/coins/nft/storage/wasm/wasm_storage.rs @@ -172,7 +172,7 @@ impl NftListStorageOps for IndexedDbNftStorage { Self::take_nft_according_to_paging_opts(nfts, max, limit, page_number) } - async fn add_nfts_to_list(&self, chain: &Chain, nfts: I, last_scanned_block: u64) -> MmResult<(), Self::Error> + async fn add_nfts_to_list(&self, chain: Chain, nfts: I, last_scanned_block: u64) -> MmResult<(), Self::Error> where I: IntoIterator + Send + 'static, I::IntoIter: Send, @@ -356,7 +356,7 @@ impl NftListStorageOps for IndexedDbNftStorage { Ok(()) } - async fn get_nfts_by_token_address(&self, chain: &Chain, token_address: String) -> MmResult, Self::Error> { + async fn get_nfts_by_token_address(&self, chain: Chain, token_address: String) -> MmResult, Self::Error> { let locked_db = self.lock_db().await?; let db_transaction = locked_db.get_inner().transaction().await?; let table = db_transaction.table::().await?; @@ -380,7 +380,7 @@ impl NftListStorageOps for IndexedDbNftStorage { possible_spam: bool, ) -> MmResult<(), Self::Error> { let locked_db = self.lock_db().await?; - let nfts: Vec = self.get_nfts_by_token_address(chain, token_address.clone()).await?; + let nfts: Vec = self.get_nfts_by_token_address(*chain, token_address.clone()).await?; let db_transaction = locked_db.get_inner().transaction().await?; let table = db_transaction.table::().await?; @@ -432,7 +432,7 @@ impl NftTransferHistoryStorageOps for IndexedDbNftStorage { Self::take_transfers_according_to_paging_opts(transfers, max, limit, page_number) } - async fn add_transfers_to_history(&self, _chain: &Chain, transfers: I) -> MmResult<(), Self::Error> + async fn add_transfers_to_history(&self, _chain: Chain, transfers: I) -> MmResult<(), Self::Error> where I: IntoIterator + Send + 'static, I::IntoIter: Send, @@ -456,7 +456,7 @@ impl NftTransferHistoryStorageOps for IndexedDbNftStorage { async fn get_transfers_from_block( &self, - chain: &Chain, + chain: Chain, from_block: u64, ) -> MmResult, Self::Error> { let locked_db = self.lock_db().await?; @@ -484,7 +484,7 @@ impl NftTransferHistoryStorageOps for IndexedDbNftStorage { async fn get_transfers_by_token_addr_id( &self, - chain: &Chain, + chain: Chain, token_address: String, token_id: BigDecimal, ) -> MmResult, Self::Error> { @@ -533,7 +533,7 @@ impl NftTransferHistoryStorageOps for IndexedDbNftStorage { ) -> MmResult<(), Self::Error> { let locked_db = self.lock_db().await?; let transfers: Vec = self - .get_transfers_by_token_addr_id(chain, transfer_meta.token_address, transfer_meta.token_id) + .get_transfers_by_token_addr_id(*chain, transfer_meta.token_address, transfer_meta.token_id) .await?; let db_transaction = locked_db.get_inner().transaction().await?; let table = db_transaction.table::().await?; @@ -590,7 +590,7 @@ impl NftTransferHistoryStorageOps for IndexedDbNftStorage { async fn get_transfers_by_token_address( &self, - chain: &Chain, + chain: Chain, token_address: String, ) -> MmResult, Self::Error> { let locked_db = self.lock_db().await?; @@ -617,7 +617,7 @@ impl NftTransferHistoryStorageOps for IndexedDbNftStorage { ) -> MmResult<(), Self::Error> { let locked_db = self.lock_db().await?; let transfers: Vec = self - .get_transfers_by_token_address(chain, token_address.clone()) + .get_transfers_by_token_address(*chain, token_address.clone()) .await?; let db_transaction = locked_db.get_inner().transaction().await?; let table = db_transaction.table::().await?; @@ -637,7 +637,7 @@ impl NftTransferHistoryStorageOps for IndexedDbNftStorage { Ok(()) } - async fn get_token_addresses(&self, chain: &Chain) -> MmResult, Self::Error> { + async fn get_token_addresses(&self, chain: Chain) -> MmResult, Self::Error> { let locked_db = self.lock_db().await?; let db_transaction = locked_db.get_inner().transaction().await?; let table = db_transaction.table::().await?; From c91fccb15648fd6b669416112bea2589433a1423 Mon Sep 17 00:00:00 2001 From: laruh Date: Wed, 6 Sep 2023 19:50:14 +0700 Subject: [PATCH 12/39] move common consts in wasm_storage --- mm2src/coins/nft/storage/wasm/wasm_storage.rs | 54 ++++++++----------- 1 file changed, 23 insertions(+), 31 deletions(-) diff --git a/mm2src/coins/nft/storage/wasm/wasm_storage.rs b/mm2src/coins/nft/storage/wasm/wasm_storage.rs index bde9923ae2..d1375e9295 100644 --- a/mm2src/coins/nft/storage/wasm/wasm_storage.rs +++ b/mm2src/coins/nft/storage/wasm/wasm_storage.rs @@ -20,6 +20,10 @@ use std::collections::HashSet; use std::num::NonZeroUsize; use std::str::FromStr; +const CHAIN_TOKEN_ADD_TOKEN_ID_INDEX: &str = "chain_token_add_token_id_index"; +const CHAIN_BLOCK_NUMBER_INDEX: &str = "chain_block_number_index"; +const CHAIN_TOKEN_ADD_INDEX: &str = "chain_token_add_index"; + #[derive(Clone)] pub struct IndexedDbNftStorage { db: SharedDb, @@ -204,7 +208,7 @@ impl NftListStorageOps for IndexedDbNftStorage { let locked_db = self.lock_db().await?; let db_transaction = locked_db.get_inner().transaction().await?; let table = db_transaction.table::().await?; - let index_keys = MultiIndex::new(NftListTable::CHAIN_TOKEN_ADD_TOKEN_ID_INDEX) + let index_keys = MultiIndex::new(CHAIN_TOKEN_ADD_TOKEN_ID_INDEX) .with_value(chain.to_string())? .with_value(&token_address)? .with_value(token_id.to_string())?; @@ -228,7 +232,7 @@ impl NftListStorageOps for IndexedDbNftStorage { let nft_table = db_transaction.table::().await?; let last_scanned_block_table = db_transaction.table::().await?; - let index_keys = MultiIndex::new(NftListTable::CHAIN_TOKEN_ADD_TOKEN_ID_INDEX) + let index_keys = MultiIndex::new(CHAIN_TOKEN_ADD_TOKEN_ID_INDEX) .with_value(chain.to_string())? .with_value(&token_address)? .with_value(token_id.to_string())?; @@ -258,7 +262,7 @@ impl NftListStorageOps for IndexedDbNftStorage { let locked_db = self.lock_db().await?; let db_transaction = locked_db.get_inner().transaction().await?; let table = db_transaction.table::().await?; - let index_keys = MultiIndex::new(NftListTable::CHAIN_TOKEN_ADD_TOKEN_ID_INDEX) + let index_keys = MultiIndex::new(CHAIN_TOKEN_ADD_TOKEN_ID_INDEX) .with_value(chain.to_string())? .with_value(&token_address)? .with_value(token_id.to_string())?; @@ -274,7 +278,7 @@ impl NftListStorageOps for IndexedDbNftStorage { let locked_db = self.lock_db().await?; let db_transaction = locked_db.get_inner().transaction().await?; let table = db_transaction.table::().await?; - let index_keys = MultiIndex::new(NftListTable::CHAIN_TOKEN_ADD_TOKEN_ID_INDEX) + let index_keys = MultiIndex::new(CHAIN_TOKEN_ADD_TOKEN_ID_INDEX) .with_value(chain.to_string())? .with_value(eth_addr_to_hex(&nft.common.token_address))? .with_value(nft.common.token_id.to_string())?; @@ -288,7 +292,7 @@ impl NftListStorageOps for IndexedDbNftStorage { let locked_db = self.lock_db().await?; let db_transaction = locked_db.get_inner().transaction().await?; let table = db_transaction.table::().await?; - get_last_block_from_table(chain, table, NftListTable::CHAIN_BLOCK_NUMBER_INDEX).await + get_last_block_from_table(chain, table, CHAIN_BLOCK_NUMBER_INDEX).await } async fn get_last_scanned_block(&self, chain: &Chain) -> MmResult, Self::Error> { @@ -312,7 +316,7 @@ impl NftListStorageOps for IndexedDbNftStorage { let nft_table = db_transaction.table::().await?; let last_scanned_block_table = db_transaction.table::().await?; - let index_keys = MultiIndex::new(NftListTable::CHAIN_TOKEN_ADD_TOKEN_ID_INDEX) + let index_keys = MultiIndex::new(CHAIN_TOKEN_ADD_TOKEN_ID_INDEX) .with_value(chain.to_string())? .with_value(eth_addr_to_hex(&nft.common.token_address))? .with_value(nft.common.token_id.to_string())?; @@ -337,7 +341,7 @@ impl NftListStorageOps for IndexedDbNftStorage { let nft_table = db_transaction.table::().await?; let last_scanned_block_table = db_transaction.table::().await?; - let index_keys = MultiIndex::new(NftListTable::CHAIN_TOKEN_ADD_TOKEN_ID_INDEX) + let index_keys = MultiIndex::new(CHAIN_TOKEN_ADD_TOKEN_ID_INDEX) .with_value(chain.to_string())? .with_value(eth_addr_to_hex(&nft.common.token_address))? .with_value(nft.common.token_id.to_string())?; @@ -361,7 +365,7 @@ impl NftListStorageOps for IndexedDbNftStorage { let db_transaction = locked_db.get_inner().transaction().await?; let table = db_transaction.table::().await?; - let index_keys = MultiIndex::new(NftListTable::CHAIN_TOKEN_ADD_INDEX) + let index_keys = MultiIndex::new(CHAIN_TOKEN_ADD_INDEX) .with_value(chain.to_string())? .with_value(&token_address)?; @@ -388,7 +392,7 @@ impl NftListStorageOps for IndexedDbNftStorage { nft.common.possible_spam = possible_spam; drop_mutability!(nft); - let index_keys = MultiIndex::new(NftListTable::CHAIN_TOKEN_ADD_TOKEN_ID_INDEX) + let index_keys = MultiIndex::new(CHAIN_TOKEN_ADD_TOKEN_ID_INDEX) .with_value(chain.to_string())? .with_value(eth_addr_to_hex(&nft.common.token_address))? .with_value(nft.common.token_id.to_string())?; @@ -451,7 +455,7 @@ impl NftTransferHistoryStorageOps for IndexedDbNftStorage { let locked_db = self.lock_db().await?; let db_transaction = locked_db.get_inner().transaction().await?; let table = db_transaction.table::().await?; - get_last_block_from_table(chain, table, NftTransferHistoryTable::CHAIN_BLOCK_NUMBER_INDEX).await + get_last_block_from_table(chain, table, CHAIN_BLOCK_NUMBER_INDEX).await } async fn get_transfers_from_block( @@ -467,7 +471,7 @@ impl NftTransferHistoryStorageOps for IndexedDbNftStorage { .only("chain", chain.to_string()) .map_err(|e| WasmNftCacheError::GetLastNftBlockError(e.to_string()))? .bound("block_number", BeBigUint::from(from_block), BeBigUint::from(u64::MAX)) - .open_cursor(NftTransferHistoryTable::CHAIN_BLOCK_NUMBER_INDEX) + .open_cursor(CHAIN_BLOCK_NUMBER_INDEX) .await .map_err(|e| WasmNftCacheError::GetLastNftBlockError(e.to_string()))? .collect() @@ -492,7 +496,7 @@ impl NftTransferHistoryStorageOps for IndexedDbNftStorage { let db_transaction = locked_db.get_inner().transaction().await?; let table = db_transaction.table::().await?; - let index_keys = MultiIndex::new(NftTransferHistoryTable::CHAIN_TOKEN_ADD_TOKEN_ID_INDEX) + let index_keys = MultiIndex::new(CHAIN_TOKEN_ADD_TOKEN_ID_INDEX) .with_value(chain.to_string())? .with_value(&token_address)? .with_value(token_id.to_string())?; @@ -597,7 +601,7 @@ impl NftTransferHistoryStorageOps for IndexedDbNftStorage { let db_transaction = locked_db.get_inner().transaction().await?; let table = db_transaction.table::().await?; - let index_keys = MultiIndex::new(NftTransferHistoryTable::CHAIN_TOKEN_ADD_INDEX) + let index_keys = MultiIndex::new(CHAIN_TOKEN_ADD_INDEX) .with_value(chain.to_string())? .with_value(&token_address)?; @@ -712,12 +716,6 @@ pub(crate) struct NftListTable { } impl NftListTable { - const CHAIN_TOKEN_ADD_TOKEN_ID_INDEX: &str = "chain_token_add_token_id_index"; - - const CHAIN_BLOCK_NUMBER_INDEX: &str = "chain_block_number_index"; - - const CHAIN_TOKEN_ADD_INDEX: &str = "chain_token_add_index"; - fn from_nft(nft: &Nft) -> WasmNftCacheResult { let details_json = json::to_value(nft).map_to_mm(|e| WasmNftCacheError::ErrorSerializing(e.to_string()))?; Ok(NftListTable { @@ -741,12 +739,12 @@ impl TableSignature for NftListTable { if is_initial_upgrade(old_version, new_version) { let table = upgrader.create_table(Self::table_name())?; table.create_multi_index( - Self::CHAIN_TOKEN_ADD_TOKEN_ID_INDEX, + CHAIN_TOKEN_ADD_TOKEN_ID_INDEX, &["chain", "token_address", "token_id"], true, )?; - table.create_multi_index(Self::CHAIN_BLOCK_NUMBER_INDEX, &["chain", "block_number"], false)?; - table.create_multi_index(Self::CHAIN_TOKEN_ADD_INDEX, &["chain", "token_address"], false)?; + table.create_multi_index(CHAIN_BLOCK_NUMBER_INDEX, &["chain", "block_number"], false)?; + table.create_multi_index(CHAIN_TOKEN_ADD_INDEX, &["chain", "token_address"], false)?; table.create_index("chain", false)?; table.create_index("block_number", false)?; } @@ -776,14 +774,8 @@ pub(crate) struct NftTransferHistoryTable { } impl NftTransferHistoryTable { - const CHAIN_TOKEN_ADD_TOKEN_ID_INDEX: &str = "chain_token_add_token_id_index"; - const CHAIN_TX_HASH_LOG_INDEX_INDEX: &str = "chain_tx_hash_log_index_index"; - const CHAIN_BLOCK_NUMBER_INDEX: &str = "chain_block_number_index"; - - const CHAIN_TOKEN_ADD_INDEX: &str = "chain_token_add_index"; - fn from_transfer_history(transfer: &NftTransferHistory) -> WasmNftCacheResult { let details_json = json::to_value(transfer).map_to_mm(|e| WasmNftCacheError::ErrorSerializing(e.to_string()))?; @@ -816,7 +808,7 @@ impl TableSignature for NftTransferHistoryTable { if is_initial_upgrade(old_version, new_version) { let table = upgrader.create_table(Self::table_name())?; table.create_multi_index( - Self::CHAIN_TOKEN_ADD_TOKEN_ID_INDEX, + CHAIN_TOKEN_ADD_TOKEN_ID_INDEX, &["chain", "token_address", "token_id"], false, )?; @@ -825,8 +817,8 @@ impl TableSignature for NftTransferHistoryTable { &["chain", "transaction_hash", "log_index"], true, )?; - table.create_multi_index(Self::CHAIN_BLOCK_NUMBER_INDEX, &["chain", "block_number"], false)?; - table.create_multi_index(Self::CHAIN_TOKEN_ADD_INDEX, &["chain", "token_address"], false)?; + table.create_multi_index(CHAIN_BLOCK_NUMBER_INDEX, &["chain", "block_number"], false)?; + table.create_multi_index(CHAIN_TOKEN_ADD_INDEX, &["chain", "token_address"], false)?; table.create_index("block_number", false)?; table.create_index("chain", false)?; } From d698708aaf923fb4bdc647b5ae1fbca2751ecdc3 Mon Sep 17 00:00:00 2001 From: laruh Date: Wed, 6 Sep 2023 22:33:05 +0700 Subject: [PATCH 13/39] use iter --- mm2src/coins/nft.rs | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index d1595a40c2..054b956206 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -155,48 +155,48 @@ pub async fn update_nft(ctx: MmArc, req: UpdateNftReq) -> MmResult<(), UpdateNft let _lock = nft_ctx.guard.lock().await; let storage = NftStorageBuilder::new(&ctx).build()?; - for chain in req.chains.into_iter() { - let transfer_history_initialized = NftTransferHistoryStorageOps::is_initialized(&storage, &chain).await?; + for chain in req.chains.iter() { + let transfer_history_initialized = NftTransferHistoryStorageOps::is_initialized(&storage, chain).await?; let from_block = if transfer_history_initialized { - let last_transfer_block = NftTransferHistoryStorageOps::get_last_block_number(&storage, &chain).await?; + let last_transfer_block = NftTransferHistoryStorageOps::get_last_block_number(&storage, chain).await?; last_transfer_block.map(|b| b + 1) } else { - NftTransferHistoryStorageOps::init(&storage, &chain).await?; + NftTransferHistoryStorageOps::init(&storage, chain).await?; None }; - let nft_transfers = get_moralis_nft_transfers(&ctx, &chain, from_block, &req.url).await?; - storage.add_transfers_to_history(chain, nft_transfers).await?; + let nft_transfers = get_moralis_nft_transfers(&ctx, chain, from_block, &req.url).await?; + storage.add_transfers_to_history(*chain, nft_transfers).await?; - let nft_block = match NftListStorageOps::get_last_block_number(&storage, &chain).await { + let nft_block = match NftListStorageOps::get_last_block_number(&storage, chain).await { Ok(Some(block)) => block, Ok(None) => { // if there are no rows in NFT LIST table we can try to get all info from moralis. - let nfts = cache_nfts_from_moralis(&ctx, &storage, &chain, &req.url).await?; - update_meta_in_transfers(&storage, &chain, nfts).await?; - update_transfers_with_empty_meta(&storage, &chain, &req.url).await?; - update_spam(&ctx, &storage, chain, &req.url_antispam).await?; + let nfts = cache_nfts_from_moralis(&ctx, &storage, chain, &req.url).await?; + update_meta_in_transfers(&storage, chain, nfts).await?; + update_transfers_with_empty_meta(&storage, chain, &req.url).await?; + update_spam(&ctx, &storage, *chain, &req.url_antispam).await?; continue; }, Err(_) => { // if there is an error, then NFT LIST table doesnt exist, so we need to cache from mroalis. - NftListStorageOps::init(&storage, &chain).await?; - let nft_list = get_moralis_nft_list(&ctx, &chain, &req.url).await?; - let last_scanned_block = NftTransferHistoryStorageOps::get_last_block_number(&storage, &chain) + NftListStorageOps::init(&storage, chain).await?; + let nft_list = get_moralis_nft_list(&ctx, chain, &req.url).await?; + let last_scanned_block = NftTransferHistoryStorageOps::get_last_block_number(&storage, chain) .await? .unwrap_or(0); storage - .add_nfts_to_list(chain, nft_list.clone(), last_scanned_block) + .add_nfts_to_list(*chain, nft_list.clone(), last_scanned_block) .await?; - update_meta_in_transfers(&storage, &chain, nft_list).await?; - update_transfers_with_empty_meta(&storage, &chain, &req.url).await?; - update_spam(&ctx, &storage, chain, &req.url_antispam).await?; + update_meta_in_transfers(&storage, chain, nft_list).await?; + update_transfers_with_empty_meta(&storage, chain, &req.url).await?; + update_spam(&ctx, &storage, *chain, &req.url_antispam).await?; continue; }, }; let scanned_block = storage - .get_last_scanned_block(&chain) + .get_last_scanned_block(chain) .await? .ok_or_else(|| UpdateNftError::LastScannedBlockNotFound { last_nft_block: nft_block.to_string(), @@ -209,9 +209,9 @@ pub async fn update_nft(ctx: MmArc, req: UpdateNftReq) -> MmResult<(), UpdateNft last_nft_block: nft_block.to_string(), }); } - update_nft_list(ctx.clone(), &storage, &chain, scanned_block + 1, &req.url).await?; - update_transfers_with_empty_meta(&storage, &chain, &req.url).await?; - update_spam(&ctx, &storage, chain, &req.url_antispam).await?; + update_nft_list(ctx.clone(), &storage, chain, scanned_block + 1, &req.url).await?; + update_transfers_with_empty_meta(&storage, chain, &req.url).await?; + update_spam(&ctx, &storage, *chain, &req.url_antispam).await?; // todo update phishing domains } Ok(()) From 220d6721fd61aa09f2605b475f804a4e12d79974 Mon Sep 17 00:00:00 2001 From: laruh Date: Wed, 6 Sep 2023 23:28:15 +0700 Subject: [PATCH 14/39] lock db after getting items --- mm2src/coins/nft/storage/wasm/wasm_storage.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mm2src/coins/nft/storage/wasm/wasm_storage.rs b/mm2src/coins/nft/storage/wasm/wasm_storage.rs index d1375e9295..c6d42dae5b 100644 --- a/mm2src/coins/nft/storage/wasm/wasm_storage.rs +++ b/mm2src/coins/nft/storage/wasm/wasm_storage.rs @@ -383,8 +383,8 @@ impl NftListStorageOps for IndexedDbNftStorage { token_address: String, possible_spam: bool, ) -> MmResult<(), Self::Error> { - let locked_db = self.lock_db().await?; let nfts: Vec = self.get_nfts_by_token_address(*chain, token_address.clone()).await?; + let locked_db = self.lock_db().await?; let db_transaction = locked_db.get_inner().transaction().await?; let table = db_transaction.table::().await?; @@ -535,10 +535,10 @@ impl NftTransferHistoryStorageOps for IndexedDbNftStorage { chain: &Chain, transfer_meta: TransferMeta, ) -> MmResult<(), Self::Error> { - let locked_db = self.lock_db().await?; let transfers: Vec = self .get_transfers_by_token_addr_id(*chain, transfer_meta.token_address, transfer_meta.token_id) .await?; + let locked_db = self.lock_db().await?; let db_transaction = locked_db.get_inner().transaction().await?; let table = db_transaction.table::().await?; for mut transfer in transfers { @@ -619,10 +619,10 @@ impl NftTransferHistoryStorageOps for IndexedDbNftStorage { token_address: String, possible_spam: bool, ) -> MmResult<(), Self::Error> { - let locked_db = self.lock_db().await?; let transfers: Vec = self .get_transfers_by_token_address(*chain, token_address.clone()) .await?; + let locked_db = self.lock_db().await?; let db_transaction = locked_db.get_inner().transaction().await?; let table = db_transaction.table::().await?; From 62796e93c3aadc2c5bcc9618cc8a5f88dfd43419 Mon Sep 17 00:00:00 2001 From: laruh Date: Thu, 7 Sep 2023 18:58:43 +0700 Subject: [PATCH 15/39] dont use several locks in wasm storage methods, limit const endpoints in nft tests, move send GET/POST reqs to mm2_net --- mm2src/coins/nft.rs | 72 ++------------- mm2src/coins/nft/nft_errors.rs | 29 +----- mm2src/coins/nft/nft_tests.rs | 89 ++++++++++++------- mm2src/coins/nft/storage/mod.rs | 3 +- mm2src/coins/nft/storage/wasm/wasm_storage.rs | 45 ++++++++-- mm2src/mm2_net/src/native_http.rs | 25 +++++- mm2src/mm2_net/src/transport.rs | 49 ++++++++++ mm2src/mm2_net/src/wasm_http.rs | 38 +++++++- 8 files changed, 211 insertions(+), 139 deletions(-) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index 054b956206..03cc172190 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -9,7 +9,7 @@ pub(crate) mod storage; #[cfg(any(test, target_arch = "wasm32"))] mod nft_tests; use crate::{coin_conf, get_my_address, MyAddressReq, WithdrawError}; -use nft_errors::{GetInfoFromUriError, GetNftInfoError, UpdateNftError}; +use nft_errors::{GetNftInfoError, UpdateNftError}; use nft_structs::{Chain, ContractType, ConvertChain, Nft, NftFromMoralis, NftList, NftListReq, NftMetadataReq, NftTransferHistory, NftTransferHistoryFromMoralis, NftTransfersReq, NftsTransferHistoryList, TransactionNftDetails, UpdateNftReq, WithdrawNftReq}; @@ -19,11 +19,11 @@ use crate::nft::nft_errors::{ProtectFromSpamError, UpdateSpamPhishingError}; use crate::nft::nft_structs::{MnemonicHQRes, NftCommon, NftCtx, NftTransferCommon, RefreshMetadataReq, SpamContractReq, SpamContractRes, TransferMeta, TransferStatus, UriMeta}; use crate::nft::storage::{NftListStorageOps, NftStorageBuilder, NftTransferHistoryStorageOps}; -use common::{parse_rfc3339_to_timestamp, APPLICATION_JSON}; +use common::parse_rfc3339_to_timestamp; use crypto::StandardHDCoinAddress; use ethereum_types::Address; -use http::header::ACCEPT; use mm2_err_handle::map_to_mm::MapToMmResult; +use mm2_net::transport::send_post_request_to_uri; use mm2_number::BigDecimal; use regex::Regex; use serde_json::Value as Json; @@ -31,10 +31,10 @@ use std::cmp::Ordering; use std::str::FromStr; #[cfg(not(target_arch = "wasm32"))] -use mm2_net::native_http::slurp_post_json; +use mm2_net::native_http::send_request_to_uri; #[cfg(target_arch = "wasm32")] -use mm2_net::wasm_http::slurp_post_json; +use mm2_net::wasm_http::send_request_to_uri; const MORALIS_API_ENDPOINT: &str = "api/v2"; /// query parameters for moralis request: The format of the token ID @@ -574,68 +574,6 @@ pub async fn withdraw_nft(ctx: MmArc, req: WithdrawNftReq) -> WithdrawNftResult } } -#[cfg(not(target_arch = "wasm32"))] -async fn send_request_to_uri(uri: &str) -> MmResult { - use http::header::HeaderValue; - use mm2_net::transport::slurp_req_body; - - let request = http::Request::builder() - .method("GET") - .uri(uri) - .header(ACCEPT, HeaderValue::from_static(APPLICATION_JSON)) - .body(hyper::Body::from(""))?; - - let (status, _header, body) = slurp_req_body(request).await?; - if !status.is_success() { - return Err(MmError::new(GetInfoFromUriError::Transport(format!( - "Response !200 from {}: {}, {}", - uri, status, body - )))); - } - Ok(body) -} - -async fn send_post_request_to_uri(uri: &str, body: String) -> MmResult, GetInfoFromUriError> { - let (status, _header, body) = slurp_post_json(uri, body).await?; - if !status.is_success() { - return Err(MmError::new(GetInfoFromUriError::Transport(format!( - "Response !200 from {}: {}", - uri, status, - )))); - } - Ok(body) -} - -#[cfg(target_arch = "wasm32")] -async fn send_request_to_uri(uri: &str) -> MmResult { - use mm2_net::wasm_http::FetchRequest; - - macro_rules! try_or { - ($exp:expr, $errtype:ident) => { - match $exp { - Ok(x) => x, - Err(e) => return Err(MmError::new(GetInfoFromUriError::$errtype(ERRL!("{:?}", e)))), - } - }; - } - - let result = FetchRequest::get(uri) - .header(ACCEPT.as_str(), APPLICATION_JSON) - .request_str() - .await; - let (status_code, response_str) = try_or!(result, Transport); - if !status_code.is_success() { - return Err(MmError::new(GetInfoFromUriError::Transport(ERRL!( - "!200: {}, {}", - status_code, - response_str - )))); - } - - let response: Json = try_or!(serde_json::from_str(&response_str), InvalidResponse); - Ok(response) -} - /// `check_moralis_ipfs_bafy` inspects a given token URI and modifies it if certain conditions are met. /// /// It checks if the URI points to the Moralis IPFS domain `"ipfs.moralis.io"` and starts with a specific path prefix `"/ipfs/bafy"`. diff --git a/mm2src/coins/nft/nft_errors.rs b/mm2src/coins/nft/nft_errors.rs index 47f7337a36..de16dd9a07 100644 --- a/mm2src/coins/nft/nft_errors.rs +++ b/mm2src/coins/nft/nft_errors.rs @@ -5,7 +5,7 @@ use common::{HttpStatusCode, ParseRfc3339Err}; use derive_more::Display; use enum_from::EnumFromStringify; use http::StatusCode; -use mm2_net::transport::SlurpError; +use mm2_net::transport::{GetInfoFromUriError, SlurpError}; use serde::{Deserialize, Serialize}; use web3::Error; @@ -233,33 +233,6 @@ impl HttpStatusCode for UpdateNftError { } } -#[derive(Clone, Debug, Deserialize, Display, EnumFromStringify, PartialEq, Serialize)] -pub enum GetInfoFromUriError { - /// `http::Error` can appear on an HTTP request [`http::Builder::build`] building. - #[from_stringify("http::Error")] - #[display(fmt = "Invalid request: {}", _0)] - InvalidRequest(String), - #[display(fmt = "Transport: {}", _0)] - Transport(String), - #[from_stringify("serde_json::Error")] - #[display(fmt = "Invalid response: {}", _0)] - InvalidResponse(String), - #[display(fmt = "Internal: {}", _0)] - Internal(String), -} - -impl From for GetInfoFromUriError { - fn from(e: SlurpError) -> Self { - let error_str = e.to_string(); - match e { - SlurpError::ErrorDeserializing { .. } => GetInfoFromUriError::InvalidResponse(error_str), - SlurpError::Transport { .. } | SlurpError::Timeout { .. } => GetInfoFromUriError::Transport(error_str), - SlurpError::InvalidRequest(_) => GetInfoFromUriError::InvalidRequest(error_str), - SlurpError::Internal(_) => GetInfoFromUriError::Internal(error_str), - } - } -} - #[derive(Clone, Debug, Deserialize, Display, EnumFromStringify, PartialEq, Serialize)] pub enum ProtectFromSpamError { #[from_stringify("regex::Error")] diff --git a/mm2src/coins/nft/nft_tests.rs b/mm2src/coins/nft/nft_tests.rs index 20dc2268eb..a8f908b61e 100644 --- a/mm2src/coins/nft/nft_tests.rs +++ b/mm2src/coins/nft/nft_tests.rs @@ -1,25 +1,19 @@ -const NFT_LIST_URL_TEST: &str = "https://moralis-proxy.komodo.earth/api/v2/0x394d86994f954ed931b86791b62fe64f4c5dac37/nft?chain=POLYGON&format=decimal"; -const NFT_HISTORY_URL_TEST: &str = "https://moralis-proxy.komodo.earth/api/v2/0x394d86994f954ed931b86791b62fe64f4c5dac37/nft/transfers?chain=POLYGON&format=decimal"; -const NFT_METADATA_URL_TEST: &str = "https://moralis-proxy.komodo.earth/api/v2/nft/0xed55e4477b795eaa9bb4bca24df42214e1a05c18/1111777?chain=POLYGON&format=decimal"; +const MORALIS_API_ENDPOINT_TEST: &str = "https://moralis-proxy.komodo.earth/api/v2"; const TEST_WALLET_ADDR_EVM: &str = "0x394d86994f954ed931b86791b62fe64f4c5dac37"; -const BLOCKLIST_WALLET_ENDPOINT: &str = - "https://nft.antispam.dragonhound.info/api/blocklist/wallet/eth/0x3eb4b12127EdC81A4d2fD49658db07005bcAd065"; -const BLOCKLIST_CONTRACT_SCAN_ENDPOINT: &str = "https://nft.antispam.dragonhound.info/api/blocklist/contract/scan"; -const BLOCKLIST_DOMAIN_SCAN_ENDPOINT: &str = "https://nft.antispam.dragonhound.info/api/blocklist/domain/scan"; +const BLOCKLIST_API_ENDPOINT: &str = "https://nft.antispam.dragonhound.info/api/blocklist"; #[cfg(all(test, not(target_arch = "wasm32")))] mod native_tests { use crate::eth::eth_addr_to_hex; use crate::nft::nft_structs::{Chain, MnemonicHQRes, NftFromMoralis, NftTransferHistoryFromMoralis, PhishingDomainReq, PhishingDomainRes, SpamContractReq, SpamContractRes, UriMeta}; - use crate::nft::nft_tests::{BLOCKLIST_CONTRACT_SCAN_ENDPOINT, BLOCKLIST_DOMAIN_SCAN_ENDPOINT, - BLOCKLIST_WALLET_ENDPOINT, NFT_HISTORY_URL_TEST, NFT_LIST_URL_TEST, - NFT_METADATA_URL_TEST, TEST_WALLET_ADDR_EVM}; + use crate::nft::nft_tests::{BLOCKLIST_API_ENDPOINT, MORALIS_API_ENDPOINT_TEST, TEST_WALLET_ADDR_EVM}; use crate::nft::storage::db_test_helpers::*; - use crate::nft::{check_and_redact_if_spam, check_moralis_ipfs_bafy, check_nft_metadata_for_spam, - send_post_request_to_uri, send_request_to_uri}; + use crate::nft::{check_and_redact_if_spam, check_moralis_ipfs_bafy, check_nft_metadata_for_spam}; use common::block_on; use ethereum_types::Address; + use mm2_net::native_http::send_request_to_uri; + use mm2_net::transport::send_post_request_to_uri; use std::str::FromStr; #[test] @@ -72,14 +66,22 @@ mod native_tests { #[test] fn test_moralis_requests() { - let response_nft_list = block_on(send_request_to_uri(NFT_LIST_URL_TEST)).unwrap(); + let uri_nft_list = format!( + "{}/{}/nft?chain=POLYGON&format=decimal", + MORALIS_API_ENDPOINT_TEST, TEST_WALLET_ADDR_EVM + ); + let response_nft_list = block_on(send_request_to_uri(uri_nft_list.as_str())).unwrap(); let nfts_list = response_nft_list["result"].as_array().unwrap(); for nft_json in nfts_list { let nft_moralis: NftFromMoralis = serde_json::from_str(&nft_json.to_string()).unwrap(); assert_eq!(TEST_WALLET_ADDR_EVM, eth_addr_to_hex(&nft_moralis.common.owner_of)); } - let response_transfer_history = block_on(send_request_to_uri(NFT_HISTORY_URL_TEST)).unwrap(); + let uri_history = format!( + "{}/{}/nft/transfers?chain=POLYGON&format=decimal", + MORALIS_API_ENDPOINT_TEST, TEST_WALLET_ADDR_EVM + ); + let response_transfer_history = block_on(send_request_to_uri(uri_history.as_str())).unwrap(); let mut transfer_list = response_transfer_history["result"].as_array().unwrap().clone(); assert!(!transfer_list.is_empty()); let first_transfer = transfer_list.remove(transfer_list.len() - 1); @@ -90,7 +92,11 @@ mod native_tests { eth_addr_to_hex(&transfer_moralis.common.to_address) ); - let response_meta = block_on(send_request_to_uri(NFT_METADATA_URL_TEST)).unwrap(); + let uri_meta = format!( + "{}/nft/0xed55e4477b795eaa9bb4bca24df42214e1a05c18/1111777?chain=POLYGON&format=decimal", + MORALIS_API_ENDPOINT_TEST + ); + let response_meta = block_on(send_request_to_uri(uri_meta.as_str())).unwrap(); let nft_moralis: NftFromMoralis = serde_json::from_str(&response_meta.to_string()).unwrap(); assert_eq!(41237364, *nft_moralis.block_number_minted.unwrap()); let token_uri = nft_moralis.common.token_uri.unwrap(); @@ -100,7 +106,11 @@ mod native_tests { #[test] fn test_antispam_api_requests() { - let mnemonichq_value = block_on(send_request_to_uri(BLOCKLIST_WALLET_ENDPOINT)).unwrap(); + let uri_mnemonichq = format!( + "{}/wallet/eth/0x3eb4b12127EdC81A4d2fD49658db07005bcAd065", + BLOCKLIST_API_ENDPOINT + ); + let mnemonichq_value = block_on(send_request_to_uri(uri_mnemonichq.as_str())).unwrap(); let mnemonichq_res: MnemonicHQRes = serde_json::from_value(mnemonichq_value).unwrap(); assert!(mnemonichq_res .spam_contracts @@ -113,7 +123,8 @@ mod native_tests { .to_string(), }; let req_json = serde_json::to_string(&req_spam).unwrap(); - let contract_scan_res = block_on(send_post_request_to_uri(BLOCKLIST_CONTRACT_SCAN_ENDPOINT, req_json)).unwrap(); + let uri_contract = format!("{}/contract/scan", BLOCKLIST_API_ENDPOINT); + let contract_scan_res = block_on(send_post_request_to_uri(uri_contract.as_str(), req_json)).unwrap(); let spam_res: SpamContractRes = serde_json::from_slice(&contract_scan_res).unwrap(); assert!(spam_res .result @@ -127,8 +138,9 @@ mod native_tests { let req_phishing = PhishingDomainReq { domains: "disposal-account-case-1f677.web.app,defi8090.vip".to_string(), }; + let uri_domain = format!("{}/domain/scan", BLOCKLIST_API_ENDPOINT); let req_json = serde_json::to_string(&req_phishing).unwrap(); - let domain_scan_res = block_on(send_post_request_to_uri(BLOCKLIST_DOMAIN_SCAN_ENDPOINT, req_json)).unwrap(); + let domain_scan_res = block_on(send_post_request_to_uri(uri_domain.as_str(), req_json)).unwrap(); let phishing_res: PhishingDomainRes = serde_json::from_slice(&domain_scan_res).unwrap(); assert!(phishing_res.result.get("disposal-account-case-1f677.web.app").unwrap()); } @@ -187,12 +199,11 @@ mod wasm_tests { use crate::eth::eth_addr_to_hex; use crate::nft::nft_structs::{Chain, MnemonicHQRes, NftFromMoralis, NftTransferHistoryFromMoralis, PhishingDomainReq, PhishingDomainRes, SpamContractReq, SpamContractRes}; - use crate::nft::nft_tests::{BLOCKLIST_CONTRACT_SCAN_ENDPOINT, BLOCKLIST_DOMAIN_SCAN_ENDPOINT, - BLOCKLIST_WALLET_ENDPOINT, NFT_HISTORY_URL_TEST, NFT_LIST_URL_TEST, - NFT_METADATA_URL_TEST, TEST_WALLET_ADDR_EVM}; + use crate::nft::nft_tests::{BLOCKLIST_API_ENDPOINT, MORALIS_API_ENDPOINT_TEST, TEST_WALLET_ADDR_EVM}; use crate::nft::storage::db_test_helpers::*; - use crate::nft::{send_post_request_to_uri, send_request_to_uri}; use ethereum_types::Address; + use mm2_net::transport::send_post_request_to_uri; + use mm2_net::wasm_http::send_request_to_uri; use std::str::FromStr; use wasm_bindgen_test::*; @@ -200,14 +211,22 @@ mod wasm_tests { #[wasm_bindgen_test] async fn test_moralis_requests() { - let response_nft_list = send_request_to_uri(NFT_LIST_URL_TEST).await.unwrap(); + let uri_nft_list = format!( + "{}/{}/nft?chain=POLYGON&format=decimal", + MORALIS_API_ENDPOINT_TEST, TEST_WALLET_ADDR_EVM + ); + let response_nft_list = send_request_to_uri(uri_nft_list.as_str()).await.unwrap(); let nfts_list = response_nft_list["result"].as_array().unwrap(); for nft_json in nfts_list { let nft_moralis: NftFromMoralis = serde_json::from_str(&nft_json.to_string()).unwrap(); assert_eq!(TEST_WALLET_ADDR_EVM, eth_addr_to_hex(&nft_moralis.common.owner_of)); } - let response_transfer_history = send_request_to_uri(NFT_HISTORY_URL_TEST).await.unwrap(); + let uri_history = format!( + "{}/{}/nft/transfers?chain=POLYGON&format=decimal", + MORALIS_API_ENDPOINT_TEST, TEST_WALLET_ADDR_EVM + ); + let response_transfer_history = send_request_to_uri(uri_history.as_str()).await.unwrap(); let mut transfer_list = response_transfer_history["result"].as_array().unwrap().clone(); assert!(!transfer_list.is_empty()); let first_transfer = transfer_list.remove(transfer_list.len() - 1); @@ -218,14 +237,22 @@ mod wasm_tests { eth_addr_to_hex(&transfer_moralis.common.to_address) ); - let response_meta = send_request_to_uri(NFT_METADATA_URL_TEST).await.unwrap(); + let uri_meta = format!( + "{}/nft/0xed55e4477b795eaa9bb4bca24df42214e1a05c18/1111777?chain=POLYGON&format=decimal", + MORALIS_API_ENDPOINT_TEST + ); + let response_meta = send_request_to_uri(uri_meta.as_str()).await.unwrap(); let nft_moralis: NftFromMoralis = serde_json::from_str(&response_meta.to_string()).unwrap(); assert_eq!(41237364, *nft_moralis.block_number_minted.unwrap()); } #[wasm_bindgen_test] async fn test_antispam_wallet_endpoint() { - let res_value = send_request_to_uri(BLOCKLIST_WALLET_ENDPOINT).await.unwrap(); + let uri_mnemonichq = format!( + "{}/wallet/eth/0x3eb4b12127EdC81A4d2fD49658db07005bcAd065", + BLOCKLIST_API_ENDPOINT + ); + let res_value = send_request_to_uri(uri_mnemonichq.as_str()).await.unwrap(); let mnemonichq_res: MnemonicHQRes = serde_json::from_value(res_value).unwrap(); assert!(mnemonichq_res .spam_contracts @@ -240,10 +267,9 @@ mod wasm_tests { addresses: "0x0ded8542fc8b2b4e781b96e99fee6406550c9b7c,0x8d1355b65da254f2cc4611453adfa8b7a13f60ee" .to_string(), }; + let uri_contract = format!("{}/contract/scan", BLOCKLIST_API_ENDPOINT); let req_json = serde_json::to_string(&req_spam).unwrap(); - let contract_scan_res = send_post_request_to_uri(BLOCKLIST_CONTRACT_SCAN_ENDPOINT, req_json) - .await - .unwrap(); + let contract_scan_res = send_post_request_to_uri(uri_contract.as_str(), req_json).await.unwrap(); let spam_res: SpamContractRes = serde_json::from_slice(&contract_scan_res).unwrap(); assert!(spam_res .result @@ -258,9 +284,8 @@ mod wasm_tests { domains: "disposal-account-case-1f677.web.app,defi8090.vip".to_string(), }; let req_json = serde_json::to_string(&req_phishing).unwrap(); - let domain_scan_res = send_post_request_to_uri(BLOCKLIST_DOMAIN_SCAN_ENDPOINT, req_json) - .await - .unwrap(); + let uri_domain = format!("{}/domain/scan", BLOCKLIST_API_ENDPOINT); + let domain_scan_res = send_post_request_to_uri(uri_domain.as_str(), req_json).await.unwrap(); let phishing_res: PhishingDomainRes = serde_json::from_slice(&domain_scan_res).unwrap(); assert!(phishing_res.result.get("disposal-account-case-1f677.web.app").unwrap()); } diff --git a/mm2src/coins/nft/storage/mod.rs b/mm2src/coins/nft/storage/mod.rs index dea9964cfc..ff5cadc2d8 100644 --- a/mm2src/coins/nft/storage/mod.rs +++ b/mm2src/coins/nft/storage/mod.rs @@ -230,9 +230,8 @@ pub(crate) struct NftDetailsJson { pub(crate) block_number_minted: Option, } -#[allow(dead_code)] /// `TransferDetailsJson` structure contains immutable parameters that are not needed for queries. -/// /// This is what `details_json` string contains in db table. +/// This is what `details_json` string contains in db table. #[derive(Clone, Debug, Default, Deserialize, Serialize)] pub(crate) struct TransferDetailsJson { pub(crate) block_hash: Option, diff --git a/mm2src/coins/nft/storage/wasm/wasm_storage.rs b/mm2src/coins/nft/storage/wasm/wasm_storage.rs index c6d42dae5b..057c08f4d0 100644 --- a/mm2src/coins/nft/storage/wasm/wasm_storage.rs +++ b/mm2src/coins/nft/storage/wasm/wasm_storage.rs @@ -383,11 +383,22 @@ impl NftListStorageOps for IndexedDbNftStorage { token_address: String, possible_spam: bool, ) -> MmResult<(), Self::Error> { - let nfts: Vec = self.get_nfts_by_token_address(*chain, token_address.clone()).await?; let locked_db = self.lock_db().await?; let db_transaction = locked_db.get_inner().transaction().await?; let table = db_transaction.table::().await?; + let index_keys = MultiIndex::new(CHAIN_TOKEN_ADD_INDEX) + .with_value(chain.to_string())? + .with_value(&token_address)?; + + let nfts: Result, _> = table + .get_items_by_multi_index(index_keys) + .await? + .into_iter() + .map(|(_item_id, item)| nft_details_from_item(item)) + .collect(); + let nfts = nfts?; + for mut nft in nfts { nft.common.possible_spam = possible_spam; drop_mutability!(nft); @@ -535,12 +546,23 @@ impl NftTransferHistoryStorageOps for IndexedDbNftStorage { chain: &Chain, transfer_meta: TransferMeta, ) -> MmResult<(), Self::Error> { - let transfers: Vec = self - .get_transfers_by_token_addr_id(*chain, transfer_meta.token_address, transfer_meta.token_id) - .await?; let locked_db = self.lock_db().await?; let db_transaction = locked_db.get_inner().transaction().await?; let table = db_transaction.table::().await?; + + let index_keys = MultiIndex::new(CHAIN_TOKEN_ADD_TOKEN_ID_INDEX) + .with_value(chain.to_string())? + .with_value(&transfer_meta.token_address)? + .with_value(transfer_meta.token_id.to_string())?; + + let transfers: Result, _> = table + .get_items_by_multi_index(index_keys) + .await? + .into_iter() + .map(|(_item_id, item)| transfer_details_from_item(item)) + .collect(); + let transfers = transfers?; + for mut transfer in transfers { transfer.token_uri = transfer_meta.token_uri.clone(); transfer.token_domain = transfer_meta.token_domain.clone(); @@ -619,13 +641,22 @@ impl NftTransferHistoryStorageOps for IndexedDbNftStorage { token_address: String, possible_spam: bool, ) -> MmResult<(), Self::Error> { - let transfers: Vec = self - .get_transfers_by_token_address(*chain, token_address.clone()) - .await?; let locked_db = self.lock_db().await?; let db_transaction = locked_db.get_inner().transaction().await?; let table = db_transaction.table::().await?; + let index_keys = MultiIndex::new(CHAIN_TOKEN_ADD_INDEX) + .with_value(chain.to_string())? + .with_value(&token_address)?; + + let transfers: Result, _> = table + .get_items_by_multi_index(index_keys) + .await? + .into_iter() + .map(|(_item_id, item)| transfer_details_from_item(item)) + .collect(); + let transfers = transfers?; + for mut transfer in transfers { transfer.common.possible_spam = possible_spam; drop_mutability!(transfer); diff --git a/mm2src/mm2_net/src/native_http.rs b/mm2src/mm2_net/src/native_http.rs index 9a79611835..924b4e8448 100644 --- a/mm2src/mm2_net/src/native_http.rs +++ b/mm2src/mm2_net/src/native_http.rs @@ -13,6 +13,7 @@ use async_trait::async_trait; use futures::channel::oneshot::Canceled; +use http::header::ACCEPT; use http::{header, HeaderValue, Request}; use hyper::client::connect::Connect; use hyper::client::ResponseFuture; @@ -23,7 +24,7 @@ use common::wio::{drive03, HYPER}; use common::APPLICATION_JSON; use mm2_err_handle::prelude::*; -use super::transport::{SlurpError, SlurpResult, SlurpResultJson}; +use super::transport::{GetInfoFromUriError, SlurpError, SlurpResult, SlurpResultJson}; /// Provides requesting http through it /// @@ -231,6 +232,28 @@ impl From for SlurpError { fn from(e: http::Error) -> Self { SlurpError::InvalidRequest(e.to_string()) } } +/// Sends a GET request to the given URI and expects a 2xx status code in response. +/// +/// # Errors +/// +/// Returns an error if the HTTP status code of the response is not in the 2xx range. +pub async fn send_request_to_uri(uri: &str) -> MmResult { + let request = http::Request::builder() + .method("GET") + .uri(uri) + .header(ACCEPT, HeaderValue::from_static(APPLICATION_JSON)) + .body(hyper::Body::from(""))?; + + let (status, _header, body) = slurp_req_body(request).await?; + if !status.is_success() { + return Err(MmError::new(GetInfoFromUriError::Transport(format!( + "Status code not in 2xx range from {}: {}, {}", + uri, status, body + )))); + } + Ok(body) +} + #[cfg(test)] mod tests { use crate::native_http::slurp_url; diff --git a/mm2src/mm2_net/src/transport.rs b/mm2src/mm2_net/src/transport.rs index 2ba04b65e1..a481432d85 100644 --- a/mm2src/mm2_net/src/transport.rs +++ b/mm2src/mm2_net/src/transport.rs @@ -84,3 +84,52 @@ pub struct GuiAuthValidation { pub timestamp_message: i64, pub signature: String, } + +#[derive(Clone, Debug, Deserialize, Display, PartialEq, Serialize)] +pub enum GetInfoFromUriError { + #[display(fmt = "Invalid request: {}", _0)] + InvalidRequest(String), + #[display(fmt = "Transport: {}", _0)] + Transport(String), + #[display(fmt = "Invalid response: {}", _0)] + InvalidResponse(String), + #[display(fmt = "Internal: {}", _0)] + Internal(String), +} + +/// `http::Error` can appear on an HTTP request [`http::Builder::build`] building. +impl From for GetInfoFromUriError { + fn from(e: http::Error) -> Self { GetInfoFromUriError::InvalidRequest(e.to_string()) } +} + +impl From for GetInfoFromUriError { + fn from(e: serde_json::Error) -> Self { GetInfoFromUriError::InvalidRequest(e.to_string()) } +} + +impl From for GetInfoFromUriError { + fn from(e: SlurpError) -> Self { + let error_str = e.to_string(); + match e { + SlurpError::ErrorDeserializing { .. } => GetInfoFromUriError::InvalidResponse(error_str), + SlurpError::Transport { .. } | SlurpError::Timeout { .. } => GetInfoFromUriError::Transport(error_str), + SlurpError::InvalidRequest(_) => GetInfoFromUriError::InvalidRequest(error_str), + SlurpError::Internal(_) => GetInfoFromUriError::Internal(error_str), + } + } +} + +/// Sends a POST request to the given URI and expects a 2xx status code in response. +/// +/// # Errors +/// +/// Returns an error if the HTTP status code of the response is not in the 2xx range. +pub async fn send_post_request_to_uri(uri: &str, body: String) -> MmResult, GetInfoFromUriError> { + let (status, _header, body) = slurp_post_json(uri, body).await?; + if !status.is_success() { + return Err(MmError::new(GetInfoFromUriError::Transport(format!( + "Status code not in 2xx range from {}: {}", + uri, status, + )))); + } + Ok(body) +} diff --git a/mm2src/mm2_net/src/wasm_http.rs b/mm2src/mm2_net/src/wasm_http.rs index 3767c2f40c..66362d60e3 100644 --- a/mm2src/mm2_net/src/wasm_http.rs +++ b/mm2src/mm2_net/src/wasm_http.rs @@ -1,11 +1,13 @@ -use crate::transport::{SlurpError, SlurpResult}; +use crate::transport::{GetInfoFromUriError, SlurpError, SlurpResult}; use common::executor::spawn_local; use common::{stringify_js_error, APPLICATION_JSON}; use futures::channel::oneshot; -use http::header::CONTENT_TYPE; +use gstuff::ERRL; +use http::header::{ACCEPT, CONTENT_TYPE}; use http::{HeaderMap, StatusCode}; use js_sys::Uint8Array; use mm2_err_handle::prelude::*; +use serde_json::Value as Json; use std::collections::HashMap; use wasm_bindgen::prelude::*; use wasm_bindgen::JsCast; @@ -282,6 +284,38 @@ impl RequestBody { } } +/// Sends a GET request to the given URI and expects a 2xx status code in response. +/// +/// # Errors +/// +/// Returns an error if the HTTP status code of the response is not in the 2xx range. +pub async fn send_request_to_uri(uri: &str) -> MmResult { + macro_rules! try_or { + ($exp:expr, $errtype:ident) => { + match $exp { + Ok(x) => x, + Err(e) => return Err(MmError::new(GetInfoFromUriError::$errtype(ERRL!("{:?}", e)))), + } + }; + } + + let result = FetchRequest::get(uri) + .header(ACCEPT.as_str(), APPLICATION_JSON) + .request_str() + .await; + let (status_code, response_str) = try_or!(result, Transport); + if !status_code.is_success() { + return Err(MmError::new(GetInfoFromUriError::Transport(ERRL!( + "Status code not in 2xx range from: {}, {}", + status_code, + response_str + )))); + } + + let response: Json = try_or!(serde_json::from_str(&response_str), InvalidResponse); + Ok(response) +} + mod tests { use super::*; use wasm_bindgen_test::*; From a47dcb38f434a972b60d1e24ce8a4810d26283fc Mon Sep 17 00:00:00 2001 From: laruh Date: Fri, 8 Sep 2023 12:18:35 +0700 Subject: [PATCH 16/39] don't scan spam contracts if there are no token addresses --- mm2src/coins/nft.rs | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index 03cc172190..d38c1b89bc 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -231,21 +231,23 @@ where return update_spam_nft_with_mnemonichq(ctx, storage, &chain, url_antispam).await; } let token_addresses = storage.get_token_addresses(chain).await?; - let addresses = token_addresses - .iter() - .map(eth_addr_to_hex) - .collect::>() - .join(","); - let spam_res = send_spam_request(&chain, url_antispam, addresses).await?; - for (address, is_spam) in spam_res.result.into_iter() { - if is_spam { - let address_hex = eth_addr_to_hex(&address); - storage - .update_nft_spam_by_token_address(&chain, address_hex.clone(), is_spam) - .await?; - storage - .update_transfer_spam_by_token_address(&chain, address_hex, is_spam) - .await?; + if !token_addresses.is_empty() { + let addresses = token_addresses + .iter() + .map(eth_addr_to_hex) + .collect::>() + .join(","); + let spam_res = send_spam_request(&chain, url_antispam, addresses).await?; + for (address, is_spam) in spam_res.result.into_iter() { + if is_spam { + let address_hex = eth_addr_to_hex(&address); + storage + .update_nft_spam_by_token_address(&chain, address_hex.clone(), is_spam) + .await?; + storage + .update_transfer_spam_by_token_address(&chain, address_hex, is_spam) + .await?; + } } } Ok(()) From 0ae9b3ef9831053f04e5dd73ff66f34181cfeda9 Mon Sep 17 00:00:00 2001 From: laruh Date: Fri, 8 Sep 2023 14:47:51 +0700 Subject: [PATCH 17/39] doc comments --- mm2src/coins/nft.rs | 46 +++++++++++++++++-- mm2src/coins/nft/nft_errors.rs | 12 +++++ mm2src/coins/nft/nft_structs.rs | 43 ++++++++++++++++- mm2src/coins/nft/storage/mod.rs | 15 ++++-- mm2src/coins/nft/storage/wasm/nft_idb.rs | 19 ++++++++ mm2src/coins/nft/storage/wasm/wasm_storage.rs | 10 ++++ mm2src/mm2_net/src/transport.rs | 1 + 7 files changed, 138 insertions(+), 8 deletions(-) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index d38c1b89bc..7bfbface2e 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -54,7 +54,24 @@ const BLOCKLIST_SCAN: &str = "scan"; /// of the generated transaction meant for transferring the NFT. On failure, it details the encountered error. pub type WithdrawNftResult = Result>; -/// `get_nft_list` function returns list of NFTs on requested chains owned by user. +/// Fetches a list of user-owned NFTs across specified chains. +/// +/// The function aggregates NFTs based on provided chains, supports pagination, and +/// allows for result limits and filters. If the `protect_from_spam` flag is true, +/// NFTs are checked and redacted for potential spam. +/// +/// # Parameters +/// +/// * `ctx`: Shared context with configurations/resources. +/// * `req`: Request specifying chains, pagination, and filters. +/// +/// # Returns +/// +/// On success, returns a detailed `NftList` containing NFTs, total count, and skipped count. +/// # Errors +/// +/// Returns `GetNftInfoError` variants for issues like invalid requests, transport failures, +/// database errors, and spam protection errors. pub async fn get_nft_list(ctx: MmArc, req: NftListReq) -> MmResult { let nft_ctx = NftCtx::from_ctx(&ctx).map_to_mm(GetNftInfoError::Internal)?; let _lock = nft_ctx.guard.lock().await; @@ -90,7 +107,11 @@ pub async fn get_nft_list(ctx: MmArc, req: NftListReq) -> MmResult`: Result containing the desired NFT or an error. +/// On success, returns the whole info about desired Nft. +/// # Errors +/// +/// Returns `GetNftInfoError` variants for issues like invalid requests, transport failures, +/// database errors, and spam protection errors. pub async fn get_nft_metadata(ctx: MmArc, req: NftMetadataReq) -> MmResult { let nft_ctx = NftCtx::from_ctx(&ctx).map_to_mm(GetNftInfoError::Internal)?; let _lock = nft_ctx.guard.lock().await; @@ -113,7 +134,26 @@ pub async fn get_nft_metadata(ctx: MmArc, req: NftMetadataReq) -> MmResult MmResult { let nft_ctx = NftCtx::from_ctx(&ctx).map_to_mm(GetNftInfoError::Internal)?; let _lock = nft_ctx.guard.lock().await; diff --git a/mm2src/coins/nft/nft_errors.rs b/mm2src/coins/nft/nft_errors.rs index de16dd9a07..115b0fbb0f 100644 --- a/mm2src/coins/nft/nft_errors.rs +++ b/mm2src/coins/nft/nft_errors.rs @@ -233,6 +233,10 @@ impl HttpStatusCode for UpdateNftError { } } +/// Enumerates the errors that can occur during spam protection operations. +/// +/// This includes issues such as regex failures during text validation and +/// serialization/deserialization problems. #[derive(Clone, Debug, Deserialize, Display, EnumFromStringify, PartialEq, Serialize)] pub enum ProtectFromSpamError { #[from_stringify("regex::Error")] @@ -241,6 +245,12 @@ pub enum ProtectFromSpamError { SerdeError(String), } +/// An enumeration representing the potential errors encountered +/// during the process of updating spam or phishing-related information. +/// +/// This error set captures various failures, from request malformation +/// to database interaction errors, providing a comprehensive view of +/// possible issues during the spam/phishing update operations. #[derive(Clone, Debug, Deserialize, Display, EnumFromStringify, PartialEq, Serialize)] pub enum UpdateSpamPhishingError { #[display(fmt = "Invalid request: {}", _0)] @@ -276,7 +286,9 @@ impl From for UpdateSpamPhishingError { fn from(err: T) -> Self { UpdateSpamPhishingError::DbError(format!("{:?}", err)) } } +/// Errors encountered when parsing a `Chain` from a string. #[derive(Debug, Display)] pub enum ParseChainTypeError { + /// The provided string does not correspond to any of the supported blockchain types. UnsupportedChainType, } diff --git a/mm2src/coins/nft/nft_structs.rs b/mm2src/coins/nft/nft_structs.rs index 66509be9e9..49835877f0 100644 --- a/mm2src/coins/nft/nft_structs.rs +++ b/mm2src/coins/nft/nft_structs.rs @@ -254,6 +254,7 @@ impl UriMeta { } /// [`NftCommon`] structure contains common fields from [`Nft`] and [`NftFromMoralis`] +/// The `possible_spam` field indicates if any potential spam has been detected. #[derive(Clone, Debug, Deserialize, Serialize)] pub struct NftCommon { pub(crate) token_address: Address, @@ -275,7 +276,8 @@ pub struct NftCommon { } /// Represents an NFT with specific chain details, contract type, and other relevant attributes. -/// This structure captures detailed information about an NFT. +/// This structure captures detailed information about an NFT. The `possible_phishing` +/// field indicates if any domains associated with the NFT have been marked as phishing. #[derive(Clone, Debug, Deserialize, Serialize)] pub struct Nft { #[serde(flatten)] @@ -389,15 +391,26 @@ pub struct TransactionNftDetails { pub(crate) transaction_type: TransactionType, } +/// Represents a request to fetch the transfer history of NFTs owned by the user across specified chains. +/// +/// The request provides options such as pagination, limiting the number of results, +/// and applying specific filters to the history. #[derive(Debug, Deserialize)] pub struct NftTransfersReq { + /// List of chains to fetch the NFT transfer history from. pub(crate) chains: Vec, + /// Optional filters to apply when fetching the NFT transfer history. pub(crate) filters: Option, + /// Parameter indicating if the maximum number of transfer records should be fetched. + /// If true, then `limit` will be ignored. #[serde(default)] pub(crate) max: bool, + /// Limit to the number of transfer records returned in a single request. #[serde(default = "ten")] pub(crate) limit: usize, + /// Page number for pagination. pub(crate) page_number: Option, + /// Flag indicating if the returned transfer history should be protected from potential spam. #[serde(default)] pub(crate) protect_from_spam: bool, } @@ -436,6 +449,7 @@ impl fmt::Display for TransferStatus { } /// [`NftTransferCommon`] structure contains common fields from [`NftTransferHistory`] and [`NftTransferHistoryFromMoralis`] +/// The `possible_spam` field indicates if any potential spam has been detected. #[derive(Clone, Debug, Deserialize, Serialize)] pub struct NftTransferCommon { pub(crate) block_hash: Option, @@ -456,6 +470,12 @@ pub struct NftTransferCommon { pub(crate) possible_spam: bool, } +/// Represents the historical transfer details of an NFT. +/// +/// Contains relevant information about the NFT transfer such as the chain, block details, +/// and contract type. Additionally, fields like `collection_name`, `token_name`, and +/// urls to metadata provide insight into the NFT's identity. The `possible_phishing` +/// field indicates if any domains associated with the NFT have been marked as phishing. #[derive(Clone, Debug, Deserialize, Serialize)] pub struct NftTransferHistory { #[serde(flatten)] @@ -475,7 +495,9 @@ pub struct NftTransferHistory { pub(crate) possible_phishing: bool, } -/// This structure is for deserializing moralis NFT transfer json to struct. +/// Represents an NFT transfer structure specifically for deserialization from Moralis's JSON response. +/// +/// This structure is adapted to the specific format provided by Moralis's API. #[derive(Debug, Deserialize)] pub(crate) struct NftTransferHistoryFromMoralis { #[serde(flatten)] @@ -485,6 +507,9 @@ pub(crate) struct NftTransferHistoryFromMoralis { pub(crate) contract_type: Option, } +/// Represents the detailed transfer history of NFTs, including the total number of transfers +/// and the number of skipped transfers. +/// It is used as a response of `get_nft_transfers` if it is successful. #[derive(Debug, Serialize)] pub struct NftsTransferHistoryList { pub(crate) transfer_history: Vec, @@ -492,6 +517,10 @@ pub struct NftsTransferHistoryList { pub(crate) total: usize, } +/// Filters that can be applied to the NFT transfer history. +/// +/// Allows filtering based on transaction type (send/receive), date range, +/// and whether to exclude spam or phishing-related transfers. #[derive(Copy, Clone, Debug, Deserialize)] pub struct NftTransferHistoryFilters { #[serde(default)] @@ -552,13 +581,23 @@ impl From for TransferMeta { } } +/// The primary context for NFT operations within the MM environment. +/// +/// This struct provides an interface for interacting with the underlying data structures +/// required for NFT operations, including guarding against concurrent accesses and +/// dealing with platform-specific storage mechanisms. pub(crate) struct NftCtx { + /// An asynchronous mutex to guard against concurrent NFT operations, ensuring data consistency. pub(crate) guard: Arc>, #[cfg(target_arch = "wasm32")] + /// Platform-specific database for caching NFT data. pub(crate) nft_cache_db: SharedDb, } impl NftCtx { + /// Create a new `NftCtx` from the given MM context. + /// + /// If an `NftCtx` instance doesn't already exist in the MM context, it gets created and cached for subsequent use. pub(crate) fn from_ctx(ctx: &MmArc) -> Result, String> { Ok(try_s!(from_ctx(&ctx.nft_ctx, move || { Ok(NftCtx { diff --git a/mm2src/coins/nft/storage/mod.rs b/mm2src/coins/nft/storage/mod.rs index ff5cadc2d8..2857e83df8 100644 --- a/mm2src/coins/nft/storage/mod.rs +++ b/mm2src/coins/nft/storage/mod.rs @@ -17,23 +17,29 @@ pub(crate) mod db_test_helpers; #[cfg(not(target_arch = "wasm32"))] pub(crate) mod sql_storage; #[cfg(target_arch = "wasm32")] pub(crate) mod wasm; +/// Represents the outcome of an attempt to remove an NFT. #[derive(Debug, PartialEq)] pub enum RemoveNftResult { + /// Indicates that the NFT was successfully removed. NftRemoved, + /// Indicates that the NFT did not exist in the storage. NftDidNotExist, } +/// Defines the standard errors that can occur in NFT storage operations pub trait NftStorageError: std::fmt::Debug + NotMmError + NotEqual + Send {} +/// Conversion trait from `NftStorageError` to `WithdrawError`. impl From for WithdrawError { fn from(err: T) -> Self { WithdrawError::DbError(format!("{:?}", err)) } } +/// Provides asynchronous operations for handling and querying NFT listings. #[async_trait] pub trait NftListStorageOps { type Error: NftStorageError; - /// Initializes tables in storage for the specified chain type. + /// Prepares the storage by initializing required tables for a specified chain type. async fn init(&self, chain: &Chain) -> MmResult<(), Self::Error>; /// Whether tables are initialized for the specified chain. @@ -101,11 +107,12 @@ pub trait NftListStorageOps { ) -> MmResult<(), Self::Error>; } +/// Provides asynchronous operations related to the history of NFT transfers. #[async_trait] pub trait NftTransferHistoryStorageOps { type Error: NftStorageError; - /// Initializes tables in storage for the specified chain type. + /// Prepares the storage by initializing required tables for a specified chain type. async fn init(&self, chain: &Chain) -> MmResult<(), Self::Error>; /// Whether tables are initialized for the specified chain. @@ -176,6 +183,7 @@ pub trait NftTransferHistoryStorageOps { async fn get_token_addresses(&self, chain: Chain) -> MmResult, Self::Error>; } +/// Represents potential errors that can occur when creating an NFT storage. #[derive(Debug, Deserialize, Display, Serialize)] pub enum CreateNftStorageError { Internal(String), @@ -190,12 +198,13 @@ impl From for WithdrawError { } /// `NftStorageBuilder` is used to create an instance that implements the [`NftListStorageOps`] -/// and [`NftTransferHistoryStorageOps`] traits.Also has guard to lock write operations. +/// and [`NftTransferHistoryStorageOps`] traits. pub struct NftStorageBuilder<'a> { ctx: &'a MmArc, } impl<'a> NftStorageBuilder<'a> { + /// Creates a new `NftStorageBuilder` instance with the provided context. #[inline] pub fn new(ctx: &MmArc) -> NftStorageBuilder<'_> { NftStorageBuilder { ctx } } diff --git a/mm2src/coins/nft/storage/wasm/nft_idb.rs b/mm2src/coins/nft/storage/wasm/nft_idb.rs index 0d7758d61a..7675ba1e6e 100644 --- a/mm2src/coins/nft/storage/wasm/nft_idb.rs +++ b/mm2src/coins/nft/storage/wasm/nft_idb.rs @@ -5,16 +5,32 @@ use mm2_db::indexed_db::{DbIdentifier, DbInstance, DbLocked, IndexedDb, IndexedD const DB_NAME: &str = "nft_cache"; const DB_VERSION: u32 = 1; + +/// Represents a locked instance of the `NftCacheIDB` database. +/// +/// This type ensures that while the database is being accessed or modified, +/// no other operations can interfere, maintaining data integrity. pub type NftCacheIDBLocked<'a> = DbLocked<'a, NftCacheIDB>; +/// Represents the IndexedDB instance specifically designed for caching NFT data. +/// +/// This struct provides an abstraction over the raw IndexedDB, offering methods +/// to interact with the database and ensuring that the database is initialized with the +/// required tables and configurations. pub struct NftCacheIDB { + /// The underlying raw IndexedDb instance. inner: IndexedDb, } #[async_trait] impl DbInstance for NftCacheIDB { + /// Return the static name of the database. fn db_name() -> &'static str { DB_NAME } + /// Initialize the `NftCacheIDB` database with the provided identifier. + /// + /// This method ensures that the database is properly set up with the correct version + /// and has the required tables (`NftListTable`, `NftTransferHistoryTable`, and `LastScannedBlockTable`). async fn init(db_id: DbIdentifier) -> InitDbResult { let inner = IndexedDbBuilder::new(db_id) .with_version(DB_VERSION) @@ -28,5 +44,8 @@ impl DbInstance for NftCacheIDB { } impl NftCacheIDB { + /// Get a reference to the underlying `IndexedDb` instance. + /// + /// This method allows for direct interaction with the raw database, bypassing any abstractions. pub(crate) fn get_inner(&self) -> &IndexedDb { &self.inner } } diff --git a/mm2src/coins/nft/storage/wasm/wasm_storage.rs b/mm2src/coins/nft/storage/wasm/wasm_storage.rs index 057c08f4d0..8f6cb9e672 100644 --- a/mm2src/coins/nft/storage/wasm/wasm_storage.rs +++ b/mm2src/coins/nft/storage/wasm/wasm_storage.rs @@ -24,12 +24,21 @@ const CHAIN_TOKEN_ADD_TOKEN_ID_INDEX: &str = "chain_token_add_token_id_index"; const CHAIN_BLOCK_NUMBER_INDEX: &str = "chain_block_number_index"; const CHAIN_TOKEN_ADD_INDEX: &str = "chain_token_add_index"; +/// Provides methods for interacting with the IndexedDB storage specifically designed for NFT data. +/// +/// This struct abstracts the intricacies of fetching and storing NFT data in the IndexedDB, +/// ensuring optimal performance and data integrity. #[derive(Clone)] pub struct IndexedDbNftStorage { + /// The underlying shared database instance for caching NFT data. db: SharedDb, } impl IndexedDbNftStorage { + /// Construct a new `IndexedDbNftStorage` using the given MM context. + /// + /// This method ensures that a proper NFT context (`NftCtx`) exists within the MM context + /// and initializes the underlying storage as required. pub fn new(ctx: &MmArc) -> MmResult { let nft_ctx = NftCtx::from_ctx(ctx).map_to_mm(CreateNftStorageError::Internal)?; Ok(IndexedDbNftStorage { @@ -37,6 +46,7 @@ impl IndexedDbNftStorage { }) } + /// Lock the underlying database to ensure exclusive access, maintaining data consistency during operations. async fn lock_db(&self) -> WasmNftCacheResult> { self.db.get_or_initialize().await.mm_err(WasmNftCacheError::from) } diff --git a/mm2src/mm2_net/src/transport.rs b/mm2src/mm2_net/src/transport.rs index a481432d85..27c039d556 100644 --- a/mm2src/mm2_net/src/transport.rs +++ b/mm2src/mm2_net/src/transport.rs @@ -85,6 +85,7 @@ pub struct GuiAuthValidation { pub signature: String, } +/// Errors encountered when making HTTP requests to fetch information from a URI. #[derive(Clone, Debug, Deserialize, Display, PartialEq, Serialize)] pub enum GetInfoFromUriError { #[display(fmt = "Invalid request: {}", _0)] From 3c697e58622b4f0522d160ec4ae89913f07ffb05 Mon Sep 17 00:00:00 2001 From: laruh Date: Tue, 12 Sep 2023 18:50:51 +0700 Subject: [PATCH 18/39] remove redundant doc com --- mm2src/coins/nft/storage/mod.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/mm2src/coins/nft/storage/mod.rs b/mm2src/coins/nft/storage/mod.rs index 2857e83df8..d58d8fda03 100644 --- a/mm2src/coins/nft/storage/mod.rs +++ b/mm2src/coins/nft/storage/mod.rs @@ -29,7 +29,6 @@ pub enum RemoveNftResult { /// Defines the standard errors that can occur in NFT storage operations pub trait NftStorageError: std::fmt::Debug + NotMmError + NotEqual + Send {} -/// Conversion trait from `NftStorageError` to `WithdrawError`. impl From for WithdrawError { fn from(err: T) -> Self { WithdrawError::DbError(format!("{:?}", err)) } } From b3c10c6791c2429ba7aa69c60a5c27a37539d6a3 Mon Sep 17 00:00:00 2001 From: laruh Date: Wed, 13 Sep 2023 20:50:57 +0700 Subject: [PATCH 19/39] use ref to chain.to_string() --- mm2src/coins/nft/storage/wasm/wasm_storage.rs | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/mm2src/coins/nft/storage/wasm/wasm_storage.rs b/mm2src/coins/nft/storage/wasm/wasm_storage.rs index 8f6cb9e672..23f900b265 100644 --- a/mm2src/coins/nft/storage/wasm/wasm_storage.rs +++ b/mm2src/coins/nft/storage/wasm/wasm_storage.rs @@ -397,8 +397,9 @@ impl NftListStorageOps for IndexedDbNftStorage { let db_transaction = locked_db.get_inner().transaction().await?; let table = db_transaction.table::().await?; + let chain_str = chain.to_string(); let index_keys = MultiIndex::new(CHAIN_TOKEN_ADD_INDEX) - .with_value(chain.to_string())? + .with_value(&chain_str)? .with_value(&token_address)?; let nfts: Result, _> = table @@ -414,7 +415,7 @@ impl NftListStorageOps for IndexedDbNftStorage { drop_mutability!(nft); let index_keys = MultiIndex::new(CHAIN_TOKEN_ADD_TOKEN_ID_INDEX) - .with_value(chain.to_string())? + .with_value(&chain_str)? .with_value(eth_addr_to_hex(&nft.common.token_address))? .with_value(nft.common.token_id.to_string())?; @@ -560,8 +561,9 @@ impl NftTransferHistoryStorageOps for IndexedDbNftStorage { let db_transaction = locked_db.get_inner().transaction().await?; let table = db_transaction.table::().await?; + let chain_str = chain.to_string(); let index_keys = MultiIndex::new(CHAIN_TOKEN_ADD_TOKEN_ID_INDEX) - .with_value(chain.to_string())? + .with_value(&chain_str)? .with_value(&transfer_meta.token_address)? .with_value(transfer_meta.token_id.to_string())?; @@ -583,7 +585,7 @@ impl NftTransferHistoryStorageOps for IndexedDbNftStorage { drop_mutability!(transfer); let index_keys = MultiIndex::new(NftTransferHistoryTable::CHAIN_TX_HASH_LOG_INDEX_INDEX) - .with_value(chain.to_string())? + .with_value(&chain_str)? .with_value(&transfer.common.transaction_hash)? .with_value(transfer.common.log_index)?; @@ -655,8 +657,9 @@ impl NftTransferHistoryStorageOps for IndexedDbNftStorage { let db_transaction = locked_db.get_inner().transaction().await?; let table = db_transaction.table::().await?; + let chain_str = chain.to_string(); let index_keys = MultiIndex::new(CHAIN_TOKEN_ADD_INDEX) - .with_value(chain.to_string())? + .with_value(&chain_str)? .with_value(&token_address)?; let transfers: Result, _> = table @@ -672,7 +675,7 @@ impl NftTransferHistoryStorageOps for IndexedDbNftStorage { drop_mutability!(transfer); let index_keys = MultiIndex::new(NftTransferHistoryTable::CHAIN_TX_HASH_LOG_INDEX_INDEX) - .with_value(chain.to_string())? + .with_value(&chain_str)? .with_value(&transfer.common.transaction_hash)? .with_value(transfer.common.log_index)?; From 58ba7aeee80f110a3970e82e5778381bd19d38be Mon Sep 17 00:00:00 2001 From: laruh Date: Thu, 14 Sep 2023 16:52:58 +0700 Subject: [PATCH 20/39] add unit tests for camo --- mm2src/coins/nft/nft_tests.rs | 40 ++++++++++++++++++++++++++++------- 1 file changed, 32 insertions(+), 8 deletions(-) diff --git a/mm2src/coins/nft/nft_tests.rs b/mm2src/coins/nft/nft_tests.rs index a8f908b61e..6b6f9d0a2e 100644 --- a/mm2src/coins/nft/nft_tests.rs +++ b/mm2src/coins/nft/nft_tests.rs @@ -1,6 +1,6 @@ const MORALIS_API_ENDPOINT_TEST: &str = "https://moralis-proxy.komodo.earth/api/v2"; const TEST_WALLET_ADDR_EVM: &str = "0x394d86994f954ed931b86791b62fe64f4c5dac37"; -const BLOCKLIST_API_ENDPOINT: &str = "https://nft.antispam.dragonhound.info/api/blocklist"; +const BLOCKLIST_API_ENDPOINT: &str = "https://nft.antispam.dragonhound.info"; #[cfg(all(test, not(target_arch = "wasm32")))] mod native_tests { @@ -107,7 +107,7 @@ mod native_tests { #[test] fn test_antispam_api_requests() { let uri_mnemonichq = format!( - "{}/wallet/eth/0x3eb4b12127EdC81A4d2fD49658db07005bcAd065", + "{}/api/blocklist/wallet/eth/0x3eb4b12127EdC81A4d2fD49658db07005bcAd065", BLOCKLIST_API_ENDPOINT ); let mnemonichq_value = block_on(send_request_to_uri(uri_mnemonichq.as_str())).unwrap(); @@ -123,7 +123,7 @@ mod native_tests { .to_string(), }; let req_json = serde_json::to_string(&req_spam).unwrap(); - let uri_contract = format!("{}/contract/scan", BLOCKLIST_API_ENDPOINT); + let uri_contract = format!("{}/api/blocklist/contract/scan", BLOCKLIST_API_ENDPOINT); let contract_scan_res = block_on(send_post_request_to_uri(uri_contract.as_str(), req_json)).unwrap(); let spam_res: SpamContractRes = serde_json::from_slice(&contract_scan_res).unwrap(); assert!(spam_res @@ -138,13 +138,25 @@ mod native_tests { let req_phishing = PhishingDomainReq { domains: "disposal-account-case-1f677.web.app,defi8090.vip".to_string(), }; - let uri_domain = format!("{}/domain/scan", BLOCKLIST_API_ENDPOINT); + let uri_domain = format!("{}/api/blocklist/domain/scan", BLOCKLIST_API_ENDPOINT); let req_json = serde_json::to_string(&req_phishing).unwrap(); let domain_scan_res = block_on(send_post_request_to_uri(uri_domain.as_str(), req_json)).unwrap(); let phishing_res: PhishingDomainRes = serde_json::from_slice(&domain_scan_res).unwrap(); assert!(phishing_res.result.get("disposal-account-case-1f677.web.app").unwrap()); } + #[test] + fn test_camo() { + let hex_token_uri = hex::encode("https://tikimetadata.s3.amazonaws.com/tiki_box.json"); + let uri_decode = format!("{}/url/decode/{}", BLOCKLIST_API_ENDPOINT, hex_token_uri); + let decode_res = block_on(send_request_to_uri(&uri_decode)).unwrap(); + let uri_meta: UriMeta = serde_json::from_value(decode_res).unwrap(); + assert_eq!( + uri_meta.raw_image_url.unwrap(), + "https://tikimetadata.s3.amazonaws.com/tiki_box.png" + ); + } + #[test] fn test_add_get_nfts() { block_on(test_add_get_nfts_impl()) } @@ -198,7 +210,7 @@ mod native_tests { mod wasm_tests { use crate::eth::eth_addr_to_hex; use crate::nft::nft_structs::{Chain, MnemonicHQRes, NftFromMoralis, NftTransferHistoryFromMoralis, - PhishingDomainReq, PhishingDomainRes, SpamContractReq, SpamContractRes}; + PhishingDomainReq, PhishingDomainRes, SpamContractReq, SpamContractRes, UriMeta}; use crate::nft::nft_tests::{BLOCKLIST_API_ENDPOINT, MORALIS_API_ENDPOINT_TEST, TEST_WALLET_ADDR_EVM}; use crate::nft::storage::db_test_helpers::*; use ethereum_types::Address; @@ -249,7 +261,7 @@ mod wasm_tests { #[wasm_bindgen_test] async fn test_antispam_wallet_endpoint() { let uri_mnemonichq = format!( - "{}/wallet/eth/0x3eb4b12127EdC81A4d2fD49658db07005bcAd065", + "{}/api/blocklist/wallet/eth/0x3eb4b12127EdC81A4d2fD49658db07005bcAd065", BLOCKLIST_API_ENDPOINT ); let res_value = send_request_to_uri(uri_mnemonichq.as_str()).await.unwrap(); @@ -267,7 +279,7 @@ mod wasm_tests { addresses: "0x0ded8542fc8b2b4e781b96e99fee6406550c9b7c,0x8d1355b65da254f2cc4611453adfa8b7a13f60ee" .to_string(), }; - let uri_contract = format!("{}/contract/scan", BLOCKLIST_API_ENDPOINT); + let uri_contract = format!("{}/api/blocklist/contract/scan", BLOCKLIST_API_ENDPOINT); let req_json = serde_json::to_string(&req_spam).unwrap(); let contract_scan_res = send_post_request_to_uri(uri_contract.as_str(), req_json).await.unwrap(); let spam_res: SpamContractRes = serde_json::from_slice(&contract_scan_res).unwrap(); @@ -284,12 +296,24 @@ mod wasm_tests { domains: "disposal-account-case-1f677.web.app,defi8090.vip".to_string(), }; let req_json = serde_json::to_string(&req_phishing).unwrap(); - let uri_domain = format!("{}/domain/scan", BLOCKLIST_API_ENDPOINT); + let uri_domain = format!("{}/api/blocklist/domain/scan", BLOCKLIST_API_ENDPOINT); let domain_scan_res = send_post_request_to_uri(uri_domain.as_str(), req_json).await.unwrap(); let phishing_res: PhishingDomainRes = serde_json::from_slice(&domain_scan_res).unwrap(); assert!(phishing_res.result.get("disposal-account-case-1f677.web.app").unwrap()); } + #[wasm_bindgen_test] + async fn test_camo() { + let hex_token_uri = hex::encode("https://tikimetadata.s3.amazonaws.com/tiki_box.json"); + let uri_decode = format!("{}/url/decode/{}", BLOCKLIST_API_ENDPOINT, hex_token_uri); + let decode_res = send_request_to_uri(&uri_decode).await.unwrap(); + let uri_meta: UriMeta = serde_json::from_value(decode_res).unwrap(); + assert_eq!( + uri_meta.raw_image_url.unwrap(), + "https://tikimetadata.s3.amazonaws.com/tiki_box.png" + ); + } + #[wasm_bindgen_test] async fn test_add_get_nfts() { test_add_get_nfts_impl().await } From 410330a5ec0022f4ba3a04d9a49fef82cc52539c Mon Sep 17 00:00:00 2001 From: laruh Date: Fri, 15 Sep 2023 16:41:09 +0700 Subject: [PATCH 21/39] send req to camo to get `UriMeta` from `token_uri` --- mm2src/coins/nft.rs | 133 ++++++++++++++++++++++++++------- mm2src/coins/nft/nft_errors.rs | 12 +++ 2 files changed, 116 insertions(+), 29 deletions(-) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index 7bfbface2e..bd2102781d 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -15,7 +15,7 @@ use nft_structs::{Chain, ContractType, ConvertChain, Nft, NftFromMoralis, NftLis TransactionNftDetails, UpdateNftReq, WithdrawNftReq}; use crate::eth::{eth_addr_to_hex, get_eth_address, withdraw_erc1155, withdraw_erc721}; -use crate::nft::nft_errors::{ProtectFromSpamError, UpdateSpamPhishingError}; +use crate::nft::nft_errors::{MetaFromUrlError, ProtectFromSpamError, UpdateSpamPhishingError}; use crate::nft::nft_structs::{MnemonicHQRes, NftCommon, NftCtx, NftTransferCommon, RefreshMetadataReq, SpamContractReq, SpamContractRes, TransferMeta, TransferStatus, UriMeta}; use crate::nft::storage::{NftListStorageOps, NftStorageBuilder, NftTransferHistoryStorageOps}; @@ -212,16 +212,16 @@ pub async fn update_nft(ctx: MmArc, req: UpdateNftReq) -> MmResult<(), UpdateNft Ok(Some(block)) => block, Ok(None) => { // if there are no rows in NFT LIST table we can try to get all info from moralis. - let nfts = cache_nfts_from_moralis(&ctx, &storage, chain, &req.url).await?; + let nfts = cache_nfts_from_moralis(&ctx, &storage, chain, &req.url, &req.url_antispam).await?; update_meta_in_transfers(&storage, chain, nfts).await?; - update_transfers_with_empty_meta(&storage, chain, &req.url).await?; + update_transfers_with_empty_meta(&storage, chain, &req.url, &req.url_antispam).await?; update_spam(&ctx, &storage, *chain, &req.url_antispam).await?; continue; }, Err(_) => { - // if there is an error, then NFT LIST table doesnt exist, so we need to cache from mroalis. + // if there is an error, then NFT LIST table doesnt exist, so we need to cache from moralis. NftListStorageOps::init(&storage, chain).await?; - let nft_list = get_moralis_nft_list(&ctx, chain, &req.url).await?; + let nft_list = get_moralis_nft_list(&ctx, chain, &req.url, &req.url_antispam).await?; let last_scanned_block = NftTransferHistoryStorageOps::get_last_block_number(&storage, chain) .await? .unwrap_or(0); @@ -229,7 +229,7 @@ pub async fn update_nft(ctx: MmArc, req: UpdateNftReq) -> MmResult<(), UpdateNft .add_nfts_to_list(*chain, nft_list.clone(), last_scanned_block) .await?; update_meta_in_transfers(&storage, chain, nft_list).await?; - update_transfers_with_empty_meta(&storage, chain, &req.url).await?; + update_transfers_with_empty_meta(&storage, chain, &req.url, &req.url_antispam).await?; update_spam(&ctx, &storage, *chain, &req.url_antispam).await?; continue; }, @@ -249,8 +249,16 @@ pub async fn update_nft(ctx: MmArc, req: UpdateNftReq) -> MmResult<(), UpdateNft last_nft_block: nft_block.to_string(), }); } - update_nft_list(ctx.clone(), &storage, chain, scanned_block + 1, &req.url).await?; - update_transfers_with_empty_meta(&storage, chain, &req.url).await?; + update_nft_list( + ctx.clone(), + &storage, + chain, + scanned_block + 1, + &req.url, + &req.url_antispam, + ) + .await?; + update_transfers_with_empty_meta(&storage, chain, &req.url, &req.url_antispam).await?; update_spam(&ctx, &storage, *chain, &req.url_antispam).await?; // todo update phishing domains } @@ -386,6 +394,7 @@ pub async fn refresh_nft_metadata(ctx: MmArc, req: RefreshMetadataReq) -> MmResu req.token_id.clone(), &req.chain, &req.url, + &req.url_antispam, ) .await?; let req_meta = NftMetadataReq { @@ -397,7 +406,12 @@ pub async fn refresh_nft_metadata(ctx: MmArc, req: RefreshMetadataReq) -> MmResu let mut nft_db = get_nft_metadata(ctx.clone(), req_meta).await?; let token_uri = check_moralis_ipfs_bafy(moralis_meta.common.token_uri.as_deref()); let token_domain = get_domain_from_url(token_uri.as_deref()); - let uri_meta = get_uri_meta(token_uri.as_deref(), moralis_meta.common.metadata.as_deref()).await; + let uri_meta = get_uri_meta( + token_uri.as_deref(), + moralis_meta.common.metadata.as_deref(), + &req.url_antispam, + ) + .await; nft_db.common.collection_name = moralis_meta.common.collection_name; nft_db.common.symbol = moralis_meta.common.symbol; nft_db.common.token_uri = token_uri; @@ -433,7 +447,12 @@ pub async fn refresh_nft_metadata(ctx: MmArc, req: RefreshMetadataReq) -> MmResu Ok(()) } -async fn get_moralis_nft_list(ctx: &MmArc, chain: &Chain, url: &Url) -> MmResult, GetNftInfoError> { +async fn get_moralis_nft_list( + ctx: &MmArc, + chain: &Chain, + url: &Url, + url_antispam: &Url, +) -> MmResult, GetNftInfoError> { let mut res_list = Vec::new(); let ticker = chain.to_ticker(); let conf = coin_conf(ctx, &ticker); @@ -464,7 +483,7 @@ async fn get_moralis_nft_list(ctx: &MmArc, chain: &Chain, url: &Url) -> MmResult Some(contract_type) => contract_type, None => continue, }; - let nft = build_nft_from_moralis(*chain, nft_moralis, contract_type).await; + let nft = build_nft_from_moralis(*chain, nft_moralis, contract_type, url_antispam).await; // collect NFTs from the page res_list.push(nft); } @@ -583,6 +602,7 @@ async fn get_moralis_metadata( token_id: BigDecimal, chain: &Chain, url: &Url, + url_antispam: &Url, ) -> MmResult { let mut uri = url.clone(); uri.set_path(MORALIS_API_ENDPOINT); @@ -602,7 +622,7 @@ async fn get_moralis_metadata( Some(contract_type) => contract_type, None => return MmError::err(GetNftInfoError::ContractTypeIsNull), }; - let nft_metadata = build_nft_from_moralis(*chain, nft_moralis, contract_type).await; + let nft_metadata = build_nft_from_moralis(*chain, nft_moralis, contract_type, url_antispam).await; Ok(nft_metadata) } @@ -638,28 +658,49 @@ fn check_moralis_ipfs_bafy(token_uri: Option<&str>) -> Option { }) } -async fn get_uri_meta(token_uri: Option<&str>, metadata: Option<&str>) -> UriMeta { +async fn get_uri_meta(token_uri: Option<&str>, metadata: Option<&str>, url_antispam: &Url) -> UriMeta { let mut uri_meta = UriMeta::default(); + // Fetching data from the URL if token_uri is provided if let Some(token_uri) = token_uri { - if let Ok(response_meta) = send_request_to_uri(token_uri).await { - if let Ok(token_uri_meta) = serde_json::from_value(response_meta) { - uri_meta = token_uri_meta; - } + if let Some(url) = construct_camo_url_with_token(token_uri, url_antispam) { + uri_meta = fetch_meta_from_url(url).await.unwrap_or_default(); } } + // Filling fields from metadata if provided if let Some(metadata) = metadata { if let Ok(meta_from_meta) = serde_json::from_str::(metadata) { - uri_meta.try_to_fill_missing_fields_from(meta_from_meta) + uri_meta.try_to_fill_missing_fields_from(meta_from_meta); } } + update_uri_moralis_ipfs_fields(&mut uri_meta); + drop_mutability!(uri_meta); + uri_meta +} + +fn construct_camo_url_with_token(token_uri: &str, url_antispam: &Url) -> Option { + let mut url = url_antispam.clone(); + url.set_path("url/decode"); + let hex_token_uri = hex::encode(token_uri); + if let Ok(mut segments) = url.path_segments_mut() { + segments.push(hex_token_uri.as_str()); + } else { + return None; + } + Some(url) +} + +async fn fetch_meta_from_url(url: Url) -> MmResult { + let response_meta = send_request_to_uri(url.as_str()).await?; + serde_json::from_value(response_meta).map_err(|e| e.into()) +} + +fn update_uri_moralis_ipfs_fields(uri_meta: &mut UriMeta) { uri_meta.image_url = check_moralis_ipfs_bafy(uri_meta.image_url.as_deref()); uri_meta.image_domain = get_domain_from_url(uri_meta.image_url.as_deref()); uri_meta.animation_url = check_moralis_ipfs_bafy(uri_meta.animation_url.as_deref()); uri_meta.animation_domain = get_domain_from_url(uri_meta.animation_url.as_deref()); uri_meta.external_url = check_moralis_ipfs_bafy(uri_meta.external_url.as_deref()); uri_meta.external_domain = get_domain_from_url(uri_meta.external_url.as_deref()); - drop_mutability!(uri_meta); - uri_meta } fn get_transfer_status(my_wallet: &str, to_address: &str) -> TransferStatus { @@ -679,6 +720,7 @@ async fn update_nft_list( chain: &Chain, scan_from_block: u64, url: &Url, + url_antispam: &Url, ) -> MmResult<(), UpdateNftError> { let transfers = storage.get_transfers_from_block(*chain, scan_from_block).await?; let req = MyAddressReq { @@ -687,7 +729,7 @@ async fn update_nft_list( }; let my_address = get_my_address(ctx.clone(), req).await?.wallet_address.to_lowercase(); for transfer in transfers.into_iter() { - handle_nft_transfer(storage, chain, url, transfer, &my_address).await?; + handle_nft_transfer(storage, chain, url, url_antispam, transfer, &my_address).await?; } Ok(()) } @@ -696,17 +738,18 @@ async fn handle_nft_transfer MmResult<(), UpdateNftError> { match (transfer.status, transfer.contract_type) { (TransferStatus::Send, ContractType::Erc721) => handle_send_erc721(storage, chain, transfer).await, (TransferStatus::Receive, ContractType::Erc721) => { - handle_receive_erc721(storage, chain, transfer, url, my_address).await + handle_receive_erc721(storage, chain, transfer, url, url_antispam, my_address).await }, (TransferStatus::Send, ContractType::Erc1155) => handle_send_erc1155(storage, chain, transfer).await, (TransferStatus::Receive, ContractType::Erc1155) => { - handle_receive_erc1155(storage, chain, transfer, url, my_address).await + handle_receive_erc1155(storage, chain, transfer, url, url_antispam, my_address).await }, } } @@ -747,6 +790,7 @@ async fn handle_receive_erc721 MmResult<(), UpdateNftError> { let nft = match storage @@ -779,6 +823,7 @@ async fn handle_receive_erc721 MmResult<(), UpdateNftError> { let nft = match storage @@ -882,11 +928,17 @@ async fn handle_receive_erc1155 MmResult, UpdateNftError> { - let nft_list = get_moralis_nft_list(ctx, chain, url).await?; + let nft_list = get_moralis_nft_list(ctx, chain, url, url_antispam).await?; let last_scanned_block = NftTransferHistoryStorageOps::get_last_block_number(storage, chain) .await? .unwrap_or(0); @@ -981,13 +1034,25 @@ where } /// `update_transfers_with_empty_meta` function updates empty metadata in transfers. -async fn update_transfers_with_empty_meta(storage: &T, chain: &Chain, url: &Url) -> MmResult<(), UpdateNftError> +async fn update_transfers_with_empty_meta( + storage: &T, + chain: &Chain, + url: &Url, + url_antispam: &Url, +) -> MmResult<(), UpdateNftError> where T: NftListStorageOps + NftTransferHistoryStorageOps, { let nft_token_addr_id = storage.get_transfers_with_empty_meta(*chain).await?; for addr_id_pair in nft_token_addr_id.into_iter() { - let nft_meta = get_moralis_metadata(addr_id_pair.token_address, addr_id_pair.token_id, chain, url).await?; + let nft_meta = get_moralis_metadata( + addr_id_pair.token_address, + addr_id_pair.token_id, + chain, + url, + url_antispam, + ) + .await?; let transfer_meta = TransferMeta::from(nft_meta); storage .update_transfers_meta_by_token_addr_id(chain, transfer_meta) @@ -1089,9 +1154,19 @@ fn check_spam_and_redact_metadata_field( } } -async fn build_nft_from_moralis(chain: Chain, nft_moralis: NftFromMoralis, contract_type: ContractType) -> Nft { +async fn build_nft_from_moralis( + chain: Chain, + nft_moralis: NftFromMoralis, + contract_type: ContractType, + url_antispam: &Url, +) -> Nft { let token_uri = check_moralis_ipfs_bafy(nft_moralis.common.token_uri.as_deref()); - let uri_meta = get_uri_meta(token_uri.as_deref(), nft_moralis.common.metadata.as_deref()).await; + let uri_meta = get_uri_meta( + token_uri.as_deref(), + nft_moralis.common.metadata.as_deref(), + url_antispam, + ) + .await; let token_domain = get_domain_from_url(token_uri.as_deref()); Nft { common: NftCommon { diff --git a/mm2src/coins/nft/nft_errors.rs b/mm2src/coins/nft/nft_errors.rs index 115b0fbb0f..13d50c83cd 100644 --- a/mm2src/coins/nft/nft_errors.rs +++ b/mm2src/coins/nft/nft_errors.rs @@ -292,3 +292,15 @@ pub enum ParseChainTypeError { /// The provided string does not correspond to any of the supported blockchain types. UnsupportedChainType, } + +#[derive(Debug, Display, EnumFromStringify)] +pub(crate) enum MetaFromUrlError { + #[from_stringify("serde_json::Error")] + #[display(fmt = "Invalid response: {}", _0)] + InvalidResponse(String), + GetInfoFromUriError(GetInfoFromUriError), +} + +impl From for MetaFromUrlError { + fn from(e: GetInfoFromUriError) -> Self { MetaFromUrlError::GetInfoFromUriError(e) } +} From 50aefefb9f8842b5869c6dc7199a86503568a741 Mon Sep 17 00:00:00 2001 From: laruh Date: Mon, 18 Sep 2023 20:41:27 +0700 Subject: [PATCH 22/39] doc com for DbInstance --- mm2src/coins/nft/storage/wasm/nft_idb.rs | 5 ----- mm2src/mm2_db/src/indexed_db/indexed_db.rs | 5 +++++ 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/mm2src/coins/nft/storage/wasm/nft_idb.rs b/mm2src/coins/nft/storage/wasm/nft_idb.rs index 7675ba1e6e..d388dd808a 100644 --- a/mm2src/coins/nft/storage/wasm/nft_idb.rs +++ b/mm2src/coins/nft/storage/wasm/nft_idb.rs @@ -24,13 +24,8 @@ pub struct NftCacheIDB { #[async_trait] impl DbInstance for NftCacheIDB { - /// Return the static name of the database. fn db_name() -> &'static str { DB_NAME } - /// Initialize the `NftCacheIDB` database with the provided identifier. - /// - /// This method ensures that the database is properly set up with the correct version - /// and has the required tables (`NftListTable`, `NftTransferHistoryTable`, and `LastScannedBlockTable`). async fn init(db_id: DbIdentifier) -> InitDbResult { let inner = IndexedDbBuilder::new(db_id) .with_version(DB_VERSION) diff --git a/mm2src/mm2_db/src/indexed_db/indexed_db.rs b/mm2src/mm2_db/src/indexed_db/indexed_db.rs index 3bad052869..dd67ea195e 100644 --- a/mm2src/mm2_db/src/indexed_db/indexed_db.rs +++ b/mm2src/mm2_db/src/indexed_db/indexed_db.rs @@ -65,10 +65,15 @@ pub trait TableSignature: DeserializeOwned + Serialize + 'static { fn on_upgrade_needed(upgrader: &DbUpgrader, old_version: u32, new_version: u32) -> OnUpgradeResult<()>; } +/// A trait representing the operations essential for initializing an IndexedDb instance. #[async_trait] pub trait DbInstance: Sized { + /// Returns the static name of the database. fn db_name() -> &'static str; + /// Initialize the database with the provided identifier. + /// This method ensures that the database is properly set up with the correct version + /// and has the required tables. async fn init(db_id: DbIdentifier) -> InitDbResult; } From 060081d85a5b26fd4aef5c51aa707e2173062912 Mon Sep 17 00:00:00 2001 From: laruh Date: Tue, 19 Sep 2023 18:38:51 +0700 Subject: [PATCH 23/39] update_phishing fnc, storage methods and unit test --- mm2src/coins/nft.rs | 81 +++++++- mm2src/coins/nft/nft_tests.rs | 23 ++- mm2src/coins/nft/storage/db_test_helpers.rs | 46 ++++- mm2src/coins/nft/storage/mod.rs | 19 ++ mm2src/coins/nft/storage/sql_storage.rs | 93 +++++++++ mm2src/coins/nft/storage/wasm/wasm_storage.rs | 183 ++++++++++++++++++ 6 files changed, 428 insertions(+), 17 deletions(-) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index bd2102781d..0cb4bce066 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -16,8 +16,9 @@ use nft_structs::{Chain, ContractType, ConvertChain, Nft, NftFromMoralis, NftLis use crate::eth::{eth_addr_to_hex, get_eth_address, withdraw_erc1155, withdraw_erc721}; use crate::nft::nft_errors::{MetaFromUrlError, ProtectFromSpamError, UpdateSpamPhishingError}; -use crate::nft::nft_structs::{MnemonicHQRes, NftCommon, NftCtx, NftTransferCommon, RefreshMetadataReq, - SpamContractReq, SpamContractRes, TransferMeta, TransferStatus, UriMeta}; +use crate::nft::nft_structs::{MnemonicHQRes, NftCommon, NftCtx, NftTransferCommon, PhishingDomainReq, + PhishingDomainRes, RefreshMetadataReq, SpamContractReq, SpamContractRes, TransferMeta, + TransferStatus, UriMeta}; use crate::nft::storage::{NftListStorageOps, NftStorageBuilder, NftTransferHistoryStorageOps}; use common::parse_rfc3339_to_timestamp; use crypto::StandardHDCoinAddress; @@ -28,6 +29,7 @@ use mm2_number::BigDecimal; use regex::Regex; use serde_json::Value as Json; use std::cmp::Ordering; +use std::collections::HashSet; use std::str::FromStr; #[cfg(not(target_arch = "wasm32"))] @@ -216,6 +218,7 @@ pub async fn update_nft(ctx: MmArc, req: UpdateNftReq) -> MmResult<(), UpdateNft update_meta_in_transfers(&storage, chain, nfts).await?; update_transfers_with_empty_meta(&storage, chain, &req.url, &req.url_antispam).await?; update_spam(&ctx, &storage, *chain, &req.url_antispam).await?; + update_phishing(&storage, chain, &req.url_antispam).await?; continue; }, Err(_) => { @@ -231,6 +234,7 @@ pub async fn update_nft(ctx: MmArc, req: UpdateNftReq) -> MmResult<(), UpdateNft update_meta_in_transfers(&storage, chain, nft_list).await?; update_transfers_with_empty_meta(&storage, chain, &req.url, &req.url_antispam).await?; update_spam(&ctx, &storage, *chain, &req.url_antispam).await?; + update_phishing(&storage, chain, &req.url_antispam).await?; continue; }, }; @@ -260,7 +264,7 @@ pub async fn update_nft(ctx: MmArc, req: UpdateNftReq) -> MmResult<(), UpdateNft .await?; update_transfers_with_empty_meta(&storage, chain, &req.url, &req.url_antispam).await?; update_spam(&ctx, &storage, *chain, &req.url_antispam).await?; - // todo update phishing domains + update_phishing(&storage, chain, &req.url_antispam).await?; } Ok(()) } @@ -301,6 +305,30 @@ where Ok(()) } +async fn update_phishing(storage: &T, chain: &Chain, url_antispam: &Url) -> MmResult<(), UpdateSpamPhishingError> +where + T: NftListStorageOps + NftTransferHistoryStorageOps, +{ + let transfer_domains = storage.get_domains(chain).await?; + let nft_domains = storage.get_animation_external_domains(chain).await?; + let domains: HashSet = transfer_domains.union(&nft_domains).cloned().collect(); + if !domains.is_empty() { + let domains = domains.into_iter().collect::>().join(","); + let domain_res = send_phishing_request(url_antispam, domains).await?; + for (domain, is_phishing) in domain_res.result.into_iter() { + if is_phishing { + storage + .update_nft_phishing_by_domain(chain, domain.clone(), is_phishing) + .await?; + storage + .update_transfer_phishing_by_domain(chain, domain, is_phishing) + .await?; + } + } + } + Ok(()) +} + /// Uses `/api/blocklist/wallet/{network}/{wallet_address}` endpoint to get **all spam contract addresses** of NFTs owned by the user. /// Currently, only the `Eth` and `Polygon` networks are supported. async fn update_spam_nft_with_mnemonichq( @@ -352,6 +380,19 @@ async fn send_spam_request( Ok(spam_res) } +/// `send_spam_request` function sends request to antispam api to scan domains for phishing. +async fn send_phishing_request( + url_antispam: &Url, + domains: String, +) -> MmResult { + let scan_contract_uri = prepare_uri_for_blocklist_endpoint(url_antispam, BLOCKLIST_DOMAIN, BLOCKLIST_SCAN)?; + let req_phishing = PhishingDomainReq { domains }; + let req_phishing_json = serde_json::to_string(&req_phishing)?; + let scan_domains_res = send_post_request_to_uri(scan_contract_uri.as_str(), req_phishing_json).await?; + let phishing_res: PhishingDomainRes = serde_json::from_slice(&scan_domains_res)?; + Ok(phishing_res) +} + /// `prepare_uri_for_blocklist_endpoint` function constructs the URI required for the antispam API request. /// It appends the required path segments to the given base URL and returns the completed URI. fn prepare_uri_for_blocklist_endpoint( @@ -412,6 +453,19 @@ pub async fn refresh_nft_metadata(ctx: MmArc, req: RefreshMetadataReq) -> MmResu &req.url_antispam, ) .await; + let mut domains = HashSet::new(); + if let Some(domain) = &token_domain { + domains.insert(domain.clone()); + } + if let Some(domain) = &uri_meta.image_domain { + domains.insert(domain.clone()); + } + if let Some(domain) = &uri_meta.animation_domain { + domains.insert(domain.clone()); + } + if let Some(domain) = &uri_meta.external_domain { + domains.insert(domain.clone()); + } nft_db.common.collection_name = moralis_meta.common.collection_name; nft_db.common.symbol = moralis_meta.common.symbol; nft_db.common.token_uri = token_uri; @@ -435,7 +489,21 @@ pub async fn refresh_nft_metadata(ctx: MmArc, req: RefreshMetadataReq) -> MmResu .update_transfer_spam_by_token_address(&req.chain, address_hex, true) .await?; } - // todo also update possible_phishing + if !domains.is_empty() { + let domain_list = domains.into_iter().collect::>().join(","); + let domain_res = send_phishing_request(&req.url_antispam, domain_list).await?; + for (domain, is_phishing) in domain_res.result.into_iter() { + if is_phishing { + nft_db.possible_phishing = true; + storage + .update_transfer_phishing_by_domain(&req.chain, domain.clone(), is_phishing) + .await?; + storage + .update_nft_phishing_by_domain(&req.chain, domain, is_phishing) + .await?; + } + } + } drop_mutability!(nft_db); storage .refresh_nft_metadata(&moralis_meta.chain, nft_db.clone()) @@ -1195,8 +1263,7 @@ async fn build_nft_from_moralis( } #[inline(always)] -fn get_domain_from_url(url: Option<&str>) -> Option { - url.as_ref() - .and_then(|uri| Url::parse(uri).ok()) +pub(crate) fn get_domain_from_url(url: Option<&str>) -> Option { + url.and_then(|uri| Url::parse(uri).ok()) .and_then(|url| url.domain().map(String::from)) } diff --git a/mm2src/coins/nft/nft_tests.rs b/mm2src/coins/nft/nft_tests.rs index 6b6f9d0a2e..e9f0df8231 100644 --- a/mm2src/coins/nft/nft_tests.rs +++ b/mm2src/coins/nft/nft_tests.rs @@ -9,7 +9,8 @@ mod native_tests { PhishingDomainReq, PhishingDomainRes, SpamContractReq, SpamContractRes, UriMeta}; use crate::nft::nft_tests::{BLOCKLIST_API_ENDPOINT, MORALIS_API_ENDPOINT_TEST, TEST_WALLET_ADDR_EVM}; use crate::nft::storage::db_test_helpers::*; - use crate::nft::{check_and_redact_if_spam, check_moralis_ipfs_bafy, check_nft_metadata_for_spam}; + use crate::nft::{check_and_redact_if_spam, check_moralis_ipfs_bafy, check_nft_metadata_for_spam, + get_domain_from_url}; use common::block_on; use ethereum_types::Address; use mm2_net::native_http::send_request_to_uri; @@ -25,6 +26,14 @@ mod native_tests { assert_eq!(expected, res_uri.unwrap()); } + #[test] + fn test_get_domain_from_url() { + let image_url = "https://public.nftstatic.com/static/nft/res/4df0a5da04174e1e9be04b22a805f605.png"; + let res_domain = get_domain_from_url(Some(image_url)); + let expected = "public.nftstatic.com"; + assert_eq!(expected, res_domain.unwrap()); + } + #[test] fn test_invalid_moralis_ipfs_link() { let uri = "example.com/bafy?1=ipfs.moralis.io&e=https://"; @@ -181,6 +190,9 @@ mod native_tests { #[test] fn test_exclude_nft_spam() { block_on(test_exclude_nft_spam_impl()) } + #[test] + fn test_get_animation_external_domains() { block_on(test_get_animation_external_domains_impl()) } + #[test] fn test_add_get_transfers() { block_on(test_add_get_transfers_impl()) } @@ -204,6 +216,9 @@ mod native_tests { #[test] fn test_exclude_transfer_spam() { block_on(test_exclude_transfer_spam_impl()) } + + #[test] + fn test_get_domains() { block_on(test_get_domains_impl()) } } #[cfg(target_arch = "wasm32")] @@ -338,6 +353,9 @@ mod wasm_tests { #[wasm_bindgen_test] async fn test_exclude_nft_spam() { test_exclude_nft_spam_impl().await } + #[wasm_bindgen_test] + async fn test_get_animation_external_domains() { test_get_animation_external_domains_impl().await } + #[wasm_bindgen_test] async fn test_add_get_transfers() { test_add_get_transfers_impl().await } @@ -361,4 +379,7 @@ mod wasm_tests { #[wasm_bindgen_test] async fn test_exclude_transfer_spam() { test_exclude_transfer_spam_impl().await } + + #[wasm_bindgen_test] + async fn test_get_domains() { test_get_domains_impl().await } } diff --git a/mm2src/coins/nft/storage/db_test_helpers.rs b/mm2src/coins/nft/storage/db_test_helpers.rs index d2e27d748b..92bd24364e 100644 --- a/mm2src/coins/nft/storage/db_test_helpers.rs +++ b/mm2src/coins/nft/storage/db_test_helpers.rs @@ -56,7 +56,7 @@ pub(crate) fn nft() -> Nft { external_url: None, external_domain: None, image_details: None, - image_domain: None, + image_domain: Some("tikimetadata.s3.amazonaws.com".to_string()), }, } } @@ -91,7 +91,7 @@ fn nft_list() -> Vec { description: Some("Born to usher in Bull markets.".to_string()), attributes: None, animation_url: None, - animation_domain: None, + animation_domain: Some("tikimetadata.s3.amazonaws.com".to_string()), external_url: None, external_domain: None, image_details: None, @@ -109,7 +109,7 @@ fn nft_list() -> Vec { collection_name: Some("Binance NFT Mystery Box-Back to Blockchain Future".to_string()), symbol: Some("BMBBBF".to_string()), token_uri: Some("https://public.nftstatic.com/static/nft/BSC/BMBBBF/214300047252".to_string()), - token_domain: None, + token_domain: Some("public.nftstatic.com".to_string()), metadata: Some( "{\"image\":\"https://public.nftstatic.com/static/nft/res/4df0a5da04174e1e9be04b22a805f605.png\"}" .to_string(), @@ -179,7 +179,7 @@ fn nft_list() -> Vec { external_url: None, external_domain: None, image_details: None, - image_domain: None, + image_domain: Some("public.nftstatic.com".to_string()), }, }; @@ -219,7 +219,7 @@ fn nft_list() -> Vec { animation_url: None, animation_domain: None, external_url: None, - external_domain: None, + external_domain: Some("public.nftstatic.com".to_string()), image_details: None, image_domain: None, }, @@ -250,7 +250,7 @@ fn nft_transfer_history() -> Vec { block_timestamp: 1677166110, contract_type: ContractType::Erc1155, token_uri: None, - token_domain: None, + token_domain: Some("tikimetadata.s3.amazonaws.com".to_string()), collection_name: None, image_url: None, image_domain: None, @@ -281,7 +281,7 @@ fn nft_transfer_history() -> Vec { block_timestamp: 1683627432, contract_type: ContractType::Erc721, token_uri: None, - token_domain: None, + token_domain: Some("public.nftstatic.com".to_string()), collection_name: None, image_url: None, image_domain: None, @@ -316,7 +316,7 @@ fn nft_transfer_history() -> Vec { token_domain: None, collection_name: None, image_url: None, - image_domain: None, + image_domain: Some("public.nftstatic.com".to_string()), token_name: None, status: TransferStatus::Receive, possible_phishing: false, @@ -347,7 +347,7 @@ fn nft_transfer_history() -> Vec { token_domain: None, collection_name: Some("Binance NFT Mystery Box-Back to Blockchain Future".to_string()), image_url: Some("https://public.nftstatic.com/static/nft/res/4df0a5da04174e1e9be04b22a805f605.png".to_string()), - image_domain: None, + image_domain: Some("tikimetadata.s3.amazonaws.com".to_string()), token_name: Some("Nebula Nodes".to_string()), status: TransferStatus::Receive, possible_phishing: false, @@ -537,6 +537,20 @@ pub(crate) async fn test_exclude_nft_spam_impl() { assert_eq!(nft_list.nfts.len(), 3); } +pub(crate) async fn test_get_animation_external_domains_impl() { + let chain = Chain::Bsc; + let storage = init_nft_list_storage(&chain).await; + let nft_list = nft_list(); + storage.add_nfts_to_list(chain, nft_list, 28056726).await.unwrap(); + + let domains = storage.get_animation_external_domains(&chain).await.unwrap(); + assert_eq!(2, domains.len()); + assert!(domains.contains("tikimetadata.s3.amazonaws.com")); + assert!(domains.contains("public.nftstatic.com")); +} + +// todo test update nft phishing + pub(crate) async fn test_add_get_transfers_impl() { let chain = Chain::Bsc; let storage = init_nft_history_storage(&chain).await; @@ -733,3 +747,17 @@ pub(crate) async fn test_exclude_transfer_spam_impl() { .unwrap(); assert_eq!(transfer_history.transfer_history.len(), 3); } + +pub(crate) async fn test_get_domains_impl() { + let chain = Chain::Bsc; + let storage = init_nft_history_storage(&chain).await; + let transfers = nft_transfer_history(); + storage.add_transfers_to_history(chain, transfers).await.unwrap(); + + let domains = storage.get_domains(&chain).await.unwrap(); + assert_eq!(2, domains.len()); + assert!(domains.contains("tikimetadata.s3.amazonaws.com")); + assert!(domains.contains("public.nftstatic.com")); +} + +// todo test update transfer phishing diff --git a/mm2src/coins/nft/storage/mod.rs b/mm2src/coins/nft/storage/mod.rs index d58d8fda03..001d7485eb 100644 --- a/mm2src/coins/nft/storage/mod.rs +++ b/mm2src/coins/nft/storage/mod.rs @@ -104,6 +104,15 @@ pub trait NftListStorageOps { token_address: String, possible_spam: bool, ) -> MmResult<(), Self::Error>; + + async fn get_animation_external_domains(&self, chain: &Chain) -> MmResult, Self::Error>; + + async fn update_nft_phishing_by_domain( + &self, + chain: &Chain, + domain: String, + possible_phishing: bool, + ) -> MmResult<(), Self::Error>; } /// Provides asynchronous operations related to the history of NFT transfers. @@ -180,6 +189,16 @@ pub trait NftTransferHistoryStorageOps { /// `get_token_addresses` return all unique token addresses. async fn get_token_addresses(&self, chain: Chain) -> MmResult, Self::Error>; + + /// `get_domains` return all unique token domain fields. + async fn get_domains(&self, chain: &Chain) -> MmResult, Self::Error>; + + async fn update_transfer_phishing_by_domain( + &self, + chain: &Chain, + domain: String, + possible_phishing: bool, + ) -> MmResult<(), Self::Error>; } /// Represents potential errors that can occur when creating an NFT storage. diff --git a/mm2src/coins/nft/storage/sql_storage.rs b/mm2src/coins/nft/storage/sql_storage.rs index ef0bb30165..794b7c58ac 100644 --- a/mm2src/coins/nft/storage/sql_storage.rs +++ b/mm2src/coins/nft/storage/sql_storage.rs @@ -879,6 +879,53 @@ impl NftListStorageOps for SqliteNftStorage { }) .await } + + async fn get_animation_external_domains(&self, chain: &Chain) -> MmResult, Self::Error> { + let selfi = self.clone(); + let table_name = chain.nft_list_table_name()?; + async_blocking(move || { + let conn = selfi.0.lock().unwrap(); + let sql_query = format!( + "SELECT DISTINCT animation_domain FROM {} + UNION + SELECT DISTINCT external_domain FROM {}", + table_name, table_name + ); + let mut stmt = conn.prepare(&sql_query)?; + let domains = stmt + .query_map([], |row| row.get::<_, Option>(0))? + .collect::, _>>()?; + let domains = domains.into_iter().flatten().collect(); + Ok(domains) + }) + .await + } + + async fn update_nft_phishing_by_domain( + &self, + chain: &Chain, + domain: String, + possible_phishing: bool, + ) -> MmResult<(), Self::Error> { + let selfi = self.clone(); + + let table_name = chain.nft_list_table_name()?; + let sql = format!( + "UPDATE {} SET possible_phishing = ?1 WHERE token_domain = ?2 + OR image_domain = ?2 OR animation_domain = ?2 OR external_domain = ?2;", + table_name + ); + + async_blocking(move || { + let mut conn = selfi.0.lock().unwrap(); + let sql_transaction = conn.transaction()?; + let params = [Some(i32::from(possible_phishing).to_string()), Some(domain)]; + sql_transaction.execute(&sql, params)?; + sql_transaction.commit()?; + Ok(()) + }) + .await + } } #[async_trait] @@ -1161,4 +1208,50 @@ impl NftTransferHistoryStorageOps for SqliteNftStorage { }) .await } + + async fn get_domains(&self, chain: &Chain) -> MmResult, Self::Error> { + let selfi = self.clone(); + let table_name = chain.transfer_history_table_name()?; + async_blocking(move || { + let conn = selfi.0.lock().unwrap(); + let sql_query = format!( + "SELECT DISTINCT token_domain FROM {} + UNION + SELECT DISTINCT image_domain FROM {}", + table_name, table_name + ); + let mut stmt = conn.prepare(&sql_query)?; + let domains = stmt + .query_map([], |row| row.get::<_, Option>(0))? + .collect::, _>>()?; + let domains = domains.into_iter().flatten().collect(); + Ok(domains) + }) + .await + } + + async fn update_transfer_phishing_by_domain( + &self, + chain: &Chain, + domain: String, + possible_phishing: bool, + ) -> MmResult<(), Self::Error> { + let selfi = self.clone(); + + let table_name = chain.transfer_history_table_name()?; + let sql = format!( + "UPDATE {} SET possible_phishing = ?1 WHERE token_domain = ?2 OR image_domain = ?2;", + table_name + ); + + async_blocking(move || { + let mut conn = selfi.0.lock().unwrap(); + let sql_transaction = conn.transaction()?; + let params = [Some(i32::from(possible_phishing).to_string()), Some(domain)]; + sql_transaction.execute(&sql, params)?; + sql_transaction.commit()?; + Ok(()) + }) + .await + } } diff --git a/mm2src/coins/nft/storage/wasm/wasm_storage.rs b/mm2src/coins/nft/storage/wasm/wasm_storage.rs index 23f900b265..b6fd51765c 100644 --- a/mm2src/coins/nft/storage/wasm/wasm_storage.rs +++ b/mm2src/coins/nft/storage/wasm/wasm_storage.rs @@ -23,6 +23,8 @@ use std::str::FromStr; const CHAIN_TOKEN_ADD_TOKEN_ID_INDEX: &str = "chain_token_add_token_id_index"; const CHAIN_BLOCK_NUMBER_INDEX: &str = "chain_block_number_index"; const CHAIN_TOKEN_ADD_INDEX: &str = "chain_token_add_index"; +const CHAIN_TOKEN_DOMAIN_INDEX: &str = "chain_token_domain_index"; +const CHAIN_IMAGE_DOMAIN_INDEX: &str = "chain_image_domain_index"; /// Provides methods for interacting with the IndexedDB storage specifically designed for NFT data. /// @@ -424,6 +426,101 @@ impl NftListStorageOps for IndexedDbNftStorage { } Ok(()) } + + async fn get_animation_external_domains(&self, chain: &Chain) -> MmResult, Self::Error> { + let locked_db = self.lock_db().await?; + let db_transaction = locked_db.get_inner().transaction().await?; + let table = db_transaction.table::().await?; + + let mut domains = HashSet::new(); + let nft_tables = table.get_items("chain", chain.to_string()).await?; + for (_item_id, nft) in nft_tables.into_iter() { + if let Some(domain) = nft.animation_domain { + domains.insert(domain); + } + if let Some(domain) = nft.external_domain { + domains.insert(domain); + } + } + Ok(domains) + } + + async fn update_nft_phishing_by_domain( + &self, + chain: &Chain, + domain: String, + possible_phishing: bool, + ) -> MmResult<(), Self::Error> { + let locked_db = self.lock_db().await?; + let db_transaction = locked_db.get_inner().transaction().await?; + let table = db_transaction.table::().await?; + + let chain_str = chain.to_string(); + let index_keys = MultiIndex::new(CHAIN_TOKEN_DOMAIN_INDEX) + .with_value(&chain_str)? + .with_value(&domain)?; + let nfts_table = table.get_items_by_multi_index(index_keys).await?; + for (_item_id, item) in nfts_table.into_iter() { + let mut nft = nft_details_from_item(item)?; + nft.possible_phishing = possible_phishing; + drop_mutability!(nft); + let nft_item = NftListTable::from_nft(&nft)?; + let index_keys = MultiIndex::new(CHAIN_TOKEN_ADD_TOKEN_ID_INDEX) + .with_value(&chain_str)? + .with_value(eth_addr_to_hex(&nft.common.token_address))? + .with_value(nft.common.token_id.to_string())?; + table.replace_item_by_unique_multi_index(index_keys, &nft_item).await?; + } + + let index_keys = MultiIndex::new(CHAIN_IMAGE_DOMAIN_INDEX) + .with_value(&chain_str)? + .with_value(&domain)?; + let nfts_table = table.get_items_by_multi_index(index_keys).await?; + for (_item_id, item) in nfts_table.into_iter() { + let mut nft = nft_details_from_item(item)?; + nft.possible_phishing = possible_phishing; + drop_mutability!(nft); + let nft_item = NftListTable::from_nft(&nft)?; + let index_keys = MultiIndex::new(CHAIN_TOKEN_ADD_TOKEN_ID_INDEX) + .with_value(&chain_str)? + .with_value(eth_addr_to_hex(&nft.common.token_address))? + .with_value(nft.common.token_id.to_string())?; + table.replace_item_by_unique_multi_index(index_keys, &nft_item).await?; + } + + let index_keys = MultiIndex::new(NftListTable::CHAIN_ANIMATION_DOMAIN_INDEX) + .with_value(&chain_str)? + .with_value(&domain)?; + let nfts_table = table.get_items_by_multi_index(index_keys).await?; + for (_item_id, item) in nfts_table.into_iter() { + let mut nft = nft_details_from_item(item)?; + nft.possible_phishing = possible_phishing; + drop_mutability!(nft); + let nft_item = NftListTable::from_nft(&nft)?; + let index_keys = MultiIndex::new(CHAIN_TOKEN_ADD_TOKEN_ID_INDEX) + .with_value(&chain_str)? + .with_value(eth_addr_to_hex(&nft.common.token_address))? + .with_value(nft.common.token_id.to_string())?; + table.replace_item_by_unique_multi_index(index_keys, &nft_item).await?; + } + + let index_keys = MultiIndex::new(NftListTable::CHAIN_EXTERNAL_DOMAIN_INDEX) + .with_value(&chain_str)? + .with_value(&domain)?; + let nfts_table = table.get_items_by_multi_index(index_keys).await?; + for (_item_id, item) in nfts_table.into_iter() { + let mut nft = nft_details_from_item(item)?; + nft.possible_phishing = possible_phishing; + drop_mutability!(nft); + let nft_item = NftListTable::from_nft(&nft)?; + let index_keys = MultiIndex::new(CHAIN_TOKEN_ADD_TOKEN_ID_INDEX) + .with_value(&chain_str)? + .with_value(eth_addr_to_hex(&nft.common.token_address))? + .with_value(nft.common.token_id.to_string())?; + table.replace_item_by_unique_multi_index(index_keys, &nft_item).await?; + } + Ok(()) + } } #[async_trait] @@ -698,6 +795,73 @@ impl NftTransferHistoryStorageOps for IndexedDbNftStorage { } Ok(token_addresses) } + + async fn get_domains(&self, chain: &Chain) -> MmResult, Self::Error> { + let locked_db = self.lock_db().await?; + let db_transaction = locked_db.get_inner().transaction().await?; + let table = db_transaction.table::().await?; + + let mut domains = HashSet::new(); + let transfer_tables = table.get_items("chain", chain.to_string()).await?; + for (_item_id, transfer) in transfer_tables.into_iter() { + if let Some(domain) = transfer.token_domain { + domains.insert(domain); + } + if let Some(domain) = transfer.image_domain { + domains.insert(domain); + } + } + Ok(domains) + } + + async fn update_transfer_phishing_by_domain( + &self, + chain: &Chain, + domain: String, + possible_phishing: bool, + ) -> MmResult<(), Self::Error> { + let locked_db = self.lock_db().await?; + let db_transaction = locked_db.get_inner().transaction().await?; + let table = db_transaction.table::().await?; + + let chain_str = chain.to_string(); + let index_keys = MultiIndex::new(CHAIN_TOKEN_DOMAIN_INDEX) + .with_value(&chain_str)? + .with_value(&domain)?; + let transfers_table = table.get_items_by_multi_index(index_keys).await?; + for (_item_id, item) in transfers_table.into_iter() { + let mut transfer = transfer_details_from_item(item)?; + transfer.possible_phishing = possible_phishing; + drop_mutability!(transfer); + let transfer_item = NftTransferHistoryTable::from_transfer_history(&transfer)?; + let index_keys = MultiIndex::new(NftTransferHistoryTable::CHAIN_TX_HASH_LOG_INDEX_INDEX) + .with_value(&chain_str)? + .with_value(&transfer.common.transaction_hash)? + .with_value(transfer.common.log_index)?; + table + .replace_item_by_unique_multi_index(index_keys, &transfer_item) + .await?; + } + + let index_keys = MultiIndex::new(CHAIN_IMAGE_DOMAIN_INDEX) + .with_value(&chain_str)? + .with_value(&domain)?; + let transfers_table = table.get_items_by_multi_index(index_keys).await?; + for (_item_id, item) in transfers_table.into_iter() { + let mut transfer = transfer_details_from_item(item)?; + transfer.possible_phishing = possible_phishing; + drop_mutability!(transfer); + let transfer_item = NftTransferHistoryTable::from_transfer_history(&transfer)?; + let index_keys = MultiIndex::new(NftTransferHistoryTable::CHAIN_TX_HASH_LOG_INDEX_INDEX) + .with_value(&chain_str)? + .with_value(&transfer.common.transaction_hash)? + .with_value(transfer.common.log_index)?; + table + .replace_item_by_unique_multi_index(index_keys, &transfer_item) + .await?; + } + Ok(()) + } } /// `get_last_block_from_table` function returns the highest block in the table related to certain blockchain type. @@ -756,10 +920,17 @@ pub(crate) struct NftListTable { contract_type: ContractType, possible_spam: bool, possible_phishing: bool, + token_domain: Option, + image_domain: Option, + animation_domain: Option, + external_domain: Option, details_json: Json, } impl NftListTable { + const CHAIN_ANIMATION_DOMAIN_INDEX: &str = "chain_animation_domain_index"; + const CHAIN_EXTERNAL_DOMAIN_INDEX: &str = "chain_external_domain_index"; + fn from_nft(nft: &Nft) -> WasmNftCacheResult { let details_json = json::to_value(nft).map_to_mm(|e| WasmNftCacheError::ErrorSerializing(e.to_string()))?; Ok(NftListTable { @@ -771,6 +942,10 @@ impl NftListTable { contract_type: nft.contract_type, possible_spam: nft.common.possible_spam, possible_phishing: nft.possible_phishing, + token_domain: nft.common.token_domain.clone(), + image_domain: nft.uri_meta.image_domain.clone(), + animation_domain: nft.uri_meta.animation_domain.clone(), + external_domain: nft.uri_meta.external_domain.clone(), details_json, }) } @@ -789,6 +964,8 @@ impl TableSignature for NftListTable { )?; table.create_multi_index(CHAIN_BLOCK_NUMBER_INDEX, &["chain", "block_number"], false)?; table.create_multi_index(CHAIN_TOKEN_ADD_INDEX, &["chain", "token_address"], false)?; + table.create_multi_index(CHAIN_TOKEN_DOMAIN_INDEX, &["chain", "token_domain"], false)?; + table.create_multi_index(CHAIN_IMAGE_DOMAIN_INDEX, &["chain", "image_domain"], false)?; table.create_index("chain", false)?; table.create_index("block_number", false)?; } @@ -809,8 +986,10 @@ pub(crate) struct NftTransferHistoryTable { status: TransferStatus, amount: String, token_uri: Option, + token_domain: Option, collection_name: Option, image_url: Option, + image_domain: Option, token_name: Option, possible_spam: bool, possible_phishing: bool, @@ -835,8 +1014,10 @@ impl NftTransferHistoryTable { status: transfer.status, amount: transfer.common.amount.to_string(), token_uri: transfer.token_uri.clone(), + token_domain: transfer.token_domain.clone(), collection_name: transfer.collection_name.clone(), image_url: transfer.image_url.clone(), + image_domain: transfer.image_domain.clone(), token_name: transfer.token_name.clone(), possible_spam: transfer.common.possible_spam, possible_phishing: transfer.possible_phishing, @@ -863,6 +1044,8 @@ impl TableSignature for NftTransferHistoryTable { )?; table.create_multi_index(CHAIN_BLOCK_NUMBER_INDEX, &["chain", "block_number"], false)?; table.create_multi_index(CHAIN_TOKEN_ADD_INDEX, &["chain", "token_address"], false)?; + table.create_multi_index(CHAIN_TOKEN_DOMAIN_INDEX, &["chain", "token_domain"], false)?; + table.create_multi_index(CHAIN_IMAGE_DOMAIN_INDEX, &["chain", "image_domain"], false)?; table.create_index("block_number", false)?; table.create_index("chain", false)?; } From 83206e4b01d641cf6076a52875506381f2a41c32 Mon Sep 17 00:00:00 2001 From: laruh Date: Wed, 20 Sep 2023 12:09:27 +0700 Subject: [PATCH 24/39] test update phishing, polish refresh metadata --- mm2src/coins/nft.rs | 106 +++++++++++++------- mm2src/coins/nft/nft_structs.rs | 1 - mm2src/coins/nft/nft_tests.rs | 12 +++ mm2src/coins/nft/storage/db_test_helpers.rs | 52 +++++++++- 4 files changed, 130 insertions(+), 41 deletions(-) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index 0cb4bce066..a2f67e21cf 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -47,7 +47,6 @@ const MORALIS_FROM_BLOCK_QUERY_NAME: &str = "from_block"; const BLOCKLIST_ENDPOINT: &str = "api/blocklist"; const BLOCKLIST_CONTRACT: &str = "contract"; -#[allow(dead_code)] const BLOCKLIST_DOMAIN: &str = "domain"; const BLOCKLIST_WALLET: &str = "wallet"; const BLOCKLIST_SCAN: &str = "scan"; @@ -430,21 +429,22 @@ pub async fn refresh_nft_metadata(ctx: MmArc, req: RefreshMetadataReq) -> MmResu let _lock = nft_ctx.guard.lock().await; let storage = NftStorageBuilder::new(&ctx).build()?; + let token_address_str = format!("{:#02x}", req.token_address); let moralis_meta = get_moralis_metadata( - format!("{:#02x}", req.token_address), + token_address_str.clone(), req.token_id.clone(), &req.chain, &req.url, &req.url_antispam, ) .await?; - let req_meta = NftMetadataReq { - token_address: req.token_address, - token_id: req.token_id, - chain: req.chain, - protect_from_spam: false, - }; - let mut nft_db = get_nft_metadata(ctx.clone(), req_meta).await?; + let mut nft_db = storage + .get_nft(&req.chain, token_address_str.clone(), req.token_id.clone()) + .await? + .ok_or_else(|| GetNftInfoError::TokenNotFoundInWallet { + token_address: token_address_str, + token_id: req.token_id.to_string(), + })?; let token_uri = check_moralis_ipfs_bafy(moralis_meta.common.token_uri.as_deref()); let token_domain = get_domain_from_url(token_uri.as_deref()); let uri_meta = get_uri_meta( @@ -453,8 +453,34 @@ pub async fn refresh_nft_metadata(ctx: MmArc, req: RefreshMetadataReq) -> MmResu &req.url_antispam, ) .await; + // Gather domains for phishing checks + let domains = gather_domains(&token_domain, &uri_meta); + nft_db.common.collection_name = moralis_meta.common.collection_name; + nft_db.common.symbol = moralis_meta.common.symbol; + nft_db.common.token_uri = token_uri; + nft_db.common.token_domain = token_domain; + nft_db.common.metadata = moralis_meta.common.metadata; + nft_db.common.last_token_uri_sync = moralis_meta.common.last_token_uri_sync; + nft_db.common.last_metadata_sync = moralis_meta.common.last_metadata_sync; + nft_db.common.possible_spam = moralis_meta.common.possible_spam; + nft_db.uri_meta = uri_meta; + refresh_possible_spam(&storage, &req.chain, &mut nft_db, &req.url_antispam).await?; + refresh_possible_phishing(&storage, &req.chain, domains, &mut nft_db, &req.url_antispam).await?; + drop_mutability!(nft_db); + storage + .refresh_nft_metadata(&moralis_meta.chain, nft_db.clone()) + .await?; + let transfer_meta = TransferMeta::from(nft_db.clone()); + storage + .update_transfers_meta_by_token_addr_id(&nft_db.chain, transfer_meta) + .await?; + Ok(()) +} + +/// Extracts domains from uri_meta and token_domain. +fn gather_domains(token_domain: &Option, uri_meta: &UriMeta) -> HashSet { let mut domains = HashSet::new(); - if let Some(domain) = &token_domain { + if let Some(domain) = token_domain { domains.insert(domain.clone()); } if let Some(domain) = &uri_meta.image_domain { @@ -466,52 +492,56 @@ pub async fn refresh_nft_metadata(ctx: MmArc, req: RefreshMetadataReq) -> MmResu if let Some(domain) = &uri_meta.external_domain { domains.insert(domain.clone()); } - nft_db.common.collection_name = moralis_meta.common.collection_name; - nft_db.common.symbol = moralis_meta.common.symbol; - nft_db.common.token_uri = token_uri; - nft_db.common.token_domain = token_domain; - nft_db.common.metadata = moralis_meta.common.metadata; - nft_db.common.last_token_uri_sync = moralis_meta.common.last_token_uri_sync; - nft_db.common.last_metadata_sync = moralis_meta.common.last_metadata_sync; - nft_db.common.possible_spam = moralis_meta.common.possible_spam; - nft_db.uri_meta = uri_meta; + domains +} - let addresses = [nft_db.common.token_address] - .iter() - .map(eth_addr_to_hex) - .collect::>() - .join(","); - let spam_res = send_spam_request(&req.chain, &req.url_antispam, addresses).await?; +/// Refreshes the `possible_spam` flag based on spam results. +async fn refresh_possible_spam( + storage: &T, + chain: &Chain, + nft_db: &mut Nft, + url_antispam: &Url, +) -> MmResult<(), UpdateNftError> +where + T: NftListStorageOps + NftTransferHistoryStorageOps, +{ + let address_hex = eth_addr_to_hex(&nft_db.common.token_address); + let spam_res = send_spam_request(chain, url_antispam, address_hex.clone()).await?; if let Some(true) = spam_res.result.get(&nft_db.common.token_address) { nft_db.common.possible_spam = true; - let address_hex = eth_addr_to_hex(&nft_db.common.token_address); storage - .update_transfer_spam_by_token_address(&req.chain, address_hex, true) + .update_transfer_spam_by_token_address(chain, address_hex, true) .await?; } + Ok(()) +} + +/// Refreshes the `possible_phishing` flag based on phishing results. +async fn refresh_possible_phishing( + storage: &T, + chain: &Chain, + domains: HashSet, + nft_db: &mut Nft, + url_antispam: &Url, +) -> MmResult<(), UpdateNftError> +where + T: NftListStorageOps + NftTransferHistoryStorageOps, +{ if !domains.is_empty() { let domain_list = domains.into_iter().collect::>().join(","); - let domain_res = send_phishing_request(&req.url_antispam, domain_list).await?; + let domain_res = send_phishing_request(url_antispam, domain_list).await?; for (domain, is_phishing) in domain_res.result.into_iter() { if is_phishing { nft_db.possible_phishing = true; storage - .update_transfer_phishing_by_domain(&req.chain, domain.clone(), is_phishing) + .update_transfer_phishing_by_domain(chain, domain.clone(), is_phishing) .await?; storage - .update_nft_phishing_by_domain(&req.chain, domain, is_phishing) + .update_nft_phishing_by_domain(chain, domain, is_phishing) .await?; } } } - drop_mutability!(nft_db); - storage - .refresh_nft_metadata(&moralis_meta.chain, nft_db.clone()) - .await?; - let transfer_meta = TransferMeta::from(nft_db.clone()); - storage - .update_transfers_meta_by_token_addr_id(&nft_db.chain, transfer_meta) - .await?; Ok(()) } diff --git a/mm2src/coins/nft/nft_structs.rs b/mm2src/coins/nft/nft_structs.rs index 49835877f0..d6d471dd78 100644 --- a/mm2src/coins/nft/nft_structs.rs +++ b/mm2src/coins/nft/nft_structs.rs @@ -625,7 +625,6 @@ pub(crate) struct SpamContractRes { pub(crate) result: HashMap, } -#[allow(dead_code)] #[derive(Debug, Deserialize)] pub(crate) struct PhishingDomainRes { pub(crate) result: HashMap, diff --git a/mm2src/coins/nft/nft_tests.rs b/mm2src/coins/nft/nft_tests.rs index e9f0df8231..6595828c32 100644 --- a/mm2src/coins/nft/nft_tests.rs +++ b/mm2src/coins/nft/nft_tests.rs @@ -193,6 +193,9 @@ mod native_tests { #[test] fn test_get_animation_external_domains() { block_on(test_get_animation_external_domains_impl()) } + #[test] + fn test_update_nft_phishing_by_domain() { block_on(test_update_nft_phishing_by_domain_impl()) } + #[test] fn test_add_get_transfers() { block_on(test_add_get_transfers_impl()) } @@ -219,6 +222,9 @@ mod native_tests { #[test] fn test_get_domains() { block_on(test_get_domains_impl()) } + + #[test] + fn test_update_transfer_phishing_by_domain() { block_on(test_update_transfer_phishing_by_domain_impl()) } } #[cfg(target_arch = "wasm32")] @@ -356,6 +362,9 @@ mod wasm_tests { #[wasm_bindgen_test] async fn test_get_animation_external_domains() { test_get_animation_external_domains_impl().await } + #[wasm_bindgen_test] + async fn test_update_nft_phishing_by_domain() { test_update_nft_phishing_by_domain_impl().await } + #[wasm_bindgen_test] async fn test_add_get_transfers() { test_add_get_transfers_impl().await } @@ -382,4 +391,7 @@ mod wasm_tests { #[wasm_bindgen_test] async fn test_get_domains() { test_get_domains_impl().await } + + #[wasm_bindgen_test] + async fn test_update_transfer_phishing_by_domain() { test_update_transfer_phishing_by_domain_impl().await } } diff --git a/mm2src/coins/nft/storage/db_test_helpers.rs b/mm2src/coins/nft/storage/db_test_helpers.rs index 92bd24364e..583a7cc64c 100644 --- a/mm2src/coins/nft/storage/db_test_helpers.rs +++ b/mm2src/coins/nft/storage/db_test_helpers.rs @@ -549,7 +549,31 @@ pub(crate) async fn test_get_animation_external_domains_impl() { assert!(domains.contains("public.nftstatic.com")); } -// todo test update nft phishing +pub(crate) async fn test_update_nft_phishing_by_domain_impl() { + let chain = Chain::Bsc; + let storage = init_nft_list_storage(&chain).await; + let nft_list = nft_list(); + storage.add_nfts_to_list(chain, nft_list, 28056726).await.unwrap(); + + let domains = vec![ + "tikimetadata.s3.amazonaws.com".to_string(), + "public.nftstatic.com".to_string(), + ]; + for domain in domains.into_iter() { + storage + .update_nft_phishing_by_domain(&chain, domain, true) + .await + .unwrap(); + } + let nfts = storage + .get_nft_list(vec![chain], true, 1, None, None) + .await + .unwrap() + .nfts; + for nft in nfts.into_iter() { + assert!(nft.possible_phishing); + } +} pub(crate) async fn test_add_get_transfers_impl() { let chain = Chain::Bsc; @@ -760,4 +784,28 @@ pub(crate) async fn test_get_domains_impl() { assert!(domains.contains("public.nftstatic.com")); } -// todo test update transfer phishing +pub(crate) async fn test_update_transfer_phishing_by_domain_impl() { + let chain = Chain::Bsc; + let storage = init_nft_history_storage(&chain).await; + let transfers = nft_transfer_history(); + storage.add_transfers_to_history(chain, transfers).await.unwrap(); + + let domains = vec![ + "tikimetadata.s3.amazonaws.com".to_string(), + "public.nftstatic.com".to_string(), + ]; + for domain in domains.into_iter() { + storage + .update_transfer_phishing_by_domain(&chain, domain, true) + .await + .unwrap(); + } + let transfers = storage + .get_transfer_history(vec![chain], true, 1, None, None) + .await + .unwrap() + .transfer_history; + for transfer in transfers.into_iter() { + assert!(transfer.possible_phishing); + } +} From 45d26ddc22a40138db24f4d6e3f15332104890d7 Mon Sep 17 00:00:00 2001 From: laruh Date: Thu, 21 Sep 2023 11:28:29 +0700 Subject: [PATCH 25/39] process text/metadata for spam links, use set_spam in update_transfers_meta_by_token_addr_id, delete update transfers meta when we remove nft from db --- mm2src/coins/nft.rs | 149 +++++++++--------- mm2src/coins/nft/nft_errors.rs | 8 +- mm2src/coins/nft/nft_tests.rs | 18 +-- mm2src/coins/nft/storage/db_test_helpers.rs | 7 +- mm2src/coins/nft/storage/mod.rs | 3 + mm2src/coins/nft/storage/sql_storage.rs | 19 +++ mm2src/coins/nft/storage/wasm/wasm_storage.rs | 4 + 7 files changed, 119 insertions(+), 89 deletions(-) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index a2f67e21cf..2dbb7ac96d 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -88,7 +88,7 @@ pub async fn get_nft_list(ctx: MmArc, req: NftListReq) -> MmResult MmResult MmResult MmResult<(), UpdateNft let nft_block = match NftListStorageOps::get_last_block_number(&storage, chain).await { Ok(Some(block)) => block, Ok(None) => { - // if there are no rows in NFT LIST table we can try to get all info from moralis. - let nfts = cache_nfts_from_moralis(&ctx, &storage, chain, &req.url, &req.url_antispam).await?; - update_meta_in_transfers(&storage, chain, nfts).await?; + // if there are no rows in NFT LIST table we can try to get nft list from moralis. + let nft_list = cache_nfts_from_moralis(&ctx, &storage, chain, &req.url, &req.url_antispam).await?; + update_meta_in_transfers(&storage, chain, nft_list).await?; update_transfers_with_empty_meta(&storage, chain, &req.url, &req.url_antispam).await?; update_spam(&ctx, &storage, *chain, &req.url_antispam).await?; update_phishing(&storage, chain, &req.url_antispam).await?; continue; }, Err(_) => { - // if there is an error, then NFT LIST table doesnt exist, so we need to cache from moralis. + // if there is an error, then NFT LIST table doesnt exist, so we need to cache nft list from moralis. NftListStorageOps::init(&storage, chain).await?; - let nft_list = get_moralis_nft_list(&ctx, chain, &req.url, &req.url_antispam).await?; - let last_scanned_block = NftTransferHistoryStorageOps::get_last_block_number(&storage, chain) - .await? - .unwrap_or(0); - storage - .add_nfts_to_list(*chain, nft_list.clone(), last_scanned_block) - .await?; + let nft_list = cache_nfts_from_moralis(&ctx, &storage, chain, &req.url, &req.url_antispam).await?; update_meta_in_transfers(&storage, chain, nft_list).await?; update_transfers_with_empty_meta(&storage, chain, &req.url, &req.url_antispam).await?; update_spam(&ctx, &storage, *chain, &req.url_antispam).await?; @@ -466,13 +460,23 @@ pub async fn refresh_nft_metadata(ctx: MmArc, req: RefreshMetadataReq) -> MmResu nft_db.uri_meta = uri_meta; refresh_possible_spam(&storage, &req.chain, &mut nft_db, &req.url_antispam).await?; refresh_possible_phishing(&storage, &req.chain, domains, &mut nft_db, &req.url_antispam).await?; - drop_mutability!(nft_db); storage .refresh_nft_metadata(&moralis_meta.chain, nft_db.clone()) .await?; - let transfer_meta = TransferMeta::from(nft_db.clone()); + update_transfer_meta_using_nft(&storage, &req.chain, &mut nft_db).await?; + Ok(()) +} + +/// The `update_transfer_meta_using_nft` function updates the transfer metadata associated with the given NFT. +/// If metadata info contains potential spam links, function sets `possible_spam` true. +async fn update_transfer_meta_using_nft(storage: &T, chain: &Chain, nft: &mut Nft) -> MmResult<(), UpdateNftError> +where + T: NftListStorageOps + NftTransferHistoryStorageOps, +{ + let set_spam = protect_from_nft_spam_links(nft, false)?; + let transfer_meta = TransferMeta::from(nft.clone()); storage - .update_transfers_meta_by_token_addr_id(&nft_db.chain, transfer_meta) + .update_transfers_meta_by_token_addr_id(chain, transfer_meta, set_spam) .await?; Ok(()) } @@ -581,7 +585,8 @@ async fn get_moralis_nft_list( Some(contract_type) => contract_type, None => continue, }; - let nft = build_nft_from_moralis(*chain, nft_moralis, contract_type, url_antispam).await; + let mut nft = build_nft_from_moralis(*chain, nft_moralis, contract_type, url_antispam).await; + protect_from_nft_spam_links(&mut nft, false)?; // collect NFTs from the page res_list.push(nft); } @@ -857,7 +862,7 @@ async fn handle_send_erc721 chain: &Chain, transfer: NftTransferHistory, ) -> MmResult<(), UpdateNftError> { - let nft_db = storage + storage .get_nft( chain, eth_addr_to_hex(&transfer.common.token_address), @@ -868,10 +873,6 @@ async fn handle_send_erc721 token_address: eth_addr_to_hex(&transfer.common.token_address), token_id: transfer.common.token_id.to_string(), })?; - let transfer_meta = TransferMeta::from(nft_db); - storage - .update_transfers_meta_by_token_addr_id(chain, transfer_meta) - .await?; storage .remove_nft_from_list( chain, @@ -891,7 +892,7 @@ async fn handle_receive_erc721 MmResult<(), UpdateNftError> { - let nft = match storage + let mut nft = match storage .get_nft( chain, eth_addr_to_hex(&transfer.common.token_address), @@ -929,6 +930,7 @@ async fn handle_receive_erc721 MmResult<(), UpdateNftError> { - let nft = match storage + let mut nft = match storage .get_nft( chain, eth_addr_to_hex(&transfer.common.token_address), @@ -1037,7 +1032,7 @@ async fn handle_receive_erc1155(storage: &T, chain: &Chain, nfts: Vec) where T: NftListStorageOps + NftTransferHistoryStorageOps, { - for nft in nfts.into_iter() { - let transfer_meta = TransferMeta::from(nft); - storage - .update_transfers_meta_by_token_addr_id(chain, transfer_meta) - .await?; + for mut nft in nfts.into_iter() { + update_transfer_meta_using_nft(storage, chain, &mut nft).await?; } Ok(()) } @@ -1143,7 +1134,7 @@ where { let nft_token_addr_id = storage.get_transfers_with_empty_meta(*chain).await?; for addr_id_pair in nft_token_addr_id.into_iter() { - let nft_meta = get_moralis_metadata( + let mut nft_meta = get_moralis_metadata( addr_id_pair.token_address, addr_id_pair.token_id, chain, @@ -1151,10 +1142,7 @@ where url_antispam, ) .await?; - let transfer_meta = TransferMeta::from(nft_meta); - storage - .update_transfers_meta_by_token_addr_id(chain, transfer_meta) - .await?; + update_transfer_meta_using_nft(storage, chain, &mut nft_meta).await?; } Ok(()) } @@ -1167,26 +1155,31 @@ fn contains_disallowed_url(text: &str) -> Result { Ok(url_regex.is_match(text)) } -/// `check_and_redact_if_spam` checks if the text contains any links. +/// `process_text_for_spam_link` checks if the text contains any links and optionally redacts it. /// It doesn't matter if the link is valid or not, as this is a spam check. -/// If text contains some link, then it is a spam. -fn check_and_redact_if_spam(text: &mut Option) -> Result { +/// If text contains some link, then function returns `true`. +fn process_text_for_spam_link(text: &mut Option, redact: bool) -> Result { match text { Some(s) if contains_disallowed_url(s)? => { - *text = Some("URL redacted for user protection".to_string()); + if redact { + *text = Some("URL redacted for user protection".to_string()); + } Ok(true) }, _ => Ok(false), } } -/// `protect_from_history_spam` function checks and redact spam in `NftTransferHistory`. +/// `protect_from_history_spam_links` function checks and redact spam in `NftTransferHistory`. /// /// `collection_name` and `token_name` in `NftTransferHistory` shouldn't contain any links, /// they must be just an arbitrary text, which represents NFT names. -fn protect_from_history_spam(transfer: &mut NftTransferHistory) -> MmResult<(), ProtectFromSpamError> { - let collection_name_spam = check_and_redact_if_spam(&mut transfer.collection_name)?; - let token_name_spam = check_and_redact_if_spam(&mut transfer.token_name)?; +fn protect_from_history_spam_links( + transfer: &mut NftTransferHistory, + redact: bool, +) -> MmResult<(), ProtectFromSpamError> { + let collection_name_spam = process_text_for_spam_link(&mut transfer.collection_name, redact)?; + let token_name_spam = process_text_for_spam_link(&mut transfer.token_name, redact)?; if collection_name_spam || token_name_spam { transfer.common.possible_spam = true; @@ -1194,58 +1187,62 @@ fn protect_from_history_spam(transfer: &mut NftTransferHistory) -> MmResult<(), Ok(()) } -/// `protect_from_nft_spam` function checks and redact spam in `Nft`. +/// `protect_from_nft_spam_links` function checks and optionally redacts spam links in `Nft`. /// /// `collection_name` and `token_name` in `Nft` shouldn't contain any links, /// they must be just an arbitrary text, which represents NFT names. /// `symbol` also must be a text or sign that represents a symbol. /// This function also checks `metadata` field for spam. -fn protect_from_nft_spam(nft: &mut Nft) -> MmResult<(), ProtectFromSpamError> { - let collection_name_spam = check_and_redact_if_spam(&mut nft.common.collection_name)?; - let symbol_spam = check_and_redact_if_spam(&mut nft.common.symbol)?; - let token_name_spam = check_and_redact_if_spam(&mut nft.uri_meta.token_name)?; - let meta_spam = check_nft_metadata_for_spam(nft)?; +fn protect_from_nft_spam_links(nft: &mut Nft, redact: bool) -> MmResult { + let collection_name_spam = process_text_for_spam_link(&mut nft.common.collection_name, redact)?; + let symbol_spam = process_text_for_spam_link(&mut nft.common.symbol, redact)?; + let token_name_spam = process_text_for_spam_link(&mut nft.uri_meta.token_name, redact)?; + let meta_spam = process_metadata_for_spam_link(nft, redact)?; if collection_name_spam || symbol_spam || token_name_spam || meta_spam { - nft.common.possible_spam = true; + return Ok(true); } - Ok(()) + Ok(false) } -/// `check_nft_metadata_for_spam` function checks and redact spam in `metadata` field from `Nft`. +/// The `process_metadata_for_spam_link` function checks and optionally redacts spam link in the `metadata` field of `Nft`. /// /// **note:** `token_name` is usually called `name` in `metadata`. -fn check_nft_metadata_for_spam(nft: &mut Nft) -> MmResult { +fn process_metadata_for_spam_link(nft: &mut Nft, redact: bool) -> MmResult { if let Some(Ok(mut metadata)) = nft .common .metadata .as_ref() .map(|t| serde_json::from_str::>(t)) { - if check_spam_and_redact_metadata_field(&mut metadata, "name")? { + let spam_detected = process_metadata_field(&mut metadata, "name", redact)?; + if redact && spam_detected { nft.common.metadata = Some(serde_json::to_string(&metadata)?); - return Ok(true); } + return Ok(spam_detected); } Ok(false) } -/// The `check_spam_and_redact_metadata_field` function scans a specified field in a JSON metadata object for potential spam. +/// The `process_metadata_field` function scans a specified field in a JSON metadata object for potential spam. /// /// This function checks the provided `metadata` map for a field matching the `field` parameter. /// If this field is found and its value contains some link, it's considered to contain spam. -/// To protect users, function redacts field containing spam link. -/// The function returns `true` if it detected spam link, or `false` otherwise. -fn check_spam_and_redact_metadata_field( +/// Depending on the `redact` flag, it will either redact the spam link or leave it as it is. +/// The function returns `true` if it detected a spam link, or `false` otherwise. +fn process_metadata_field( metadata: &mut serde_json::Map, field: &str, + redact: bool, ) -> MmResult { match metadata.get(field).and_then(|v| v.as_str()) { Some(text) if contains_disallowed_url(text)? => { - metadata.insert( - field.to_string(), - serde_json::Value::String("URL redacted for user protection".to_string()), - ); + if redact { + metadata.insert( + field.to_string(), + serde_json::Value::String("URL redacted for user protection".to_string()), + ); + } Ok(true) }, _ => Ok(false), diff --git a/mm2src/coins/nft/nft_errors.rs b/mm2src/coins/nft/nft_errors.rs index 13d50c83cd..941348db67 100644 --- a/mm2src/coins/nft/nft_errors.rs +++ b/mm2src/coins/nft/nft_errors.rs @@ -183,6 +183,7 @@ pub enum UpdateNftError { GetInfoFromUriError(GetInfoFromUriError), #[from_stringify("serde_json::Error")] SerdeError(String), + ProtectFromSpamError(ProtectFromSpamError), } impl From for UpdateNftError { @@ -213,6 +214,10 @@ impl From for UpdateNftError { fn from(e: GetInfoFromUriError) -> Self { UpdateNftError::GetInfoFromUriError(e) } } +impl From for UpdateNftError { + fn from(e: ProtectFromSpamError) -> Self { UpdateNftError::ProtectFromSpamError(e) } +} + impl HttpStatusCode for UpdateNftError { fn status_code(&self) -> StatusCode { match self { @@ -228,7 +233,8 @@ impl HttpStatusCode for UpdateNftError { | UpdateNftError::InvalidHexString(_) | UpdateNftError::UpdateSpamPhishingError(_) | UpdateNftError::GetInfoFromUriError(_) - | UpdateNftError::SerdeError(_) => StatusCode::INTERNAL_SERVER_ERROR, + | UpdateNftError::SerdeError(_) + | UpdateNftError::ProtectFromSpamError(_) => StatusCode::INTERNAL_SERVER_ERROR, } } } diff --git a/mm2src/coins/nft/nft_tests.rs b/mm2src/coins/nft/nft_tests.rs index 6595828c32..be00a4d47c 100644 --- a/mm2src/coins/nft/nft_tests.rs +++ b/mm2src/coins/nft/nft_tests.rs @@ -9,8 +9,8 @@ mod native_tests { PhishingDomainReq, PhishingDomainRes, SpamContractReq, SpamContractRes, UriMeta}; use crate::nft::nft_tests::{BLOCKLIST_API_ENDPOINT, MORALIS_API_ENDPOINT_TEST, TEST_WALLET_ADDR_EVM}; use crate::nft::storage::db_test_helpers::*; - use crate::nft::{check_and_redact_if_spam, check_moralis_ipfs_bafy, check_nft_metadata_for_spam, - get_domain_from_url}; + use crate::nft::{check_moralis_ipfs_bafy, get_domain_from_url, process_metadata_for_spam_link, + process_text_for_spam_link}; use common::block_on; use ethereum_types::Address; use mm2_net::native_http::send_request_to_uri; @@ -42,33 +42,33 @@ mod native_tests { } #[test] - fn test_check_for_spam() { + fn test_check_for_spam_links() { let mut spam_text = Some("https://arweave.net".to_string()); - assert!(check_and_redact_if_spam(&mut spam_text).unwrap()); + assert!(process_text_for_spam_link(&mut spam_text, true).unwrap()); let url_redacted = "URL redacted for user protection"; assert_eq!(url_redacted, spam_text.unwrap()); let mut spam_text = Some("ftp://123path ".to_string()); - assert!(check_and_redact_if_spam(&mut spam_text).unwrap()); + assert!(process_text_for_spam_link(&mut spam_text, true).unwrap()); let url_redacted = "URL redacted for user protection"; assert_eq!(url_redacted, spam_text.unwrap()); let mut spam_text = Some("/192.168.1.1/some.example.org?type=A".to_string()); - assert!(check_and_redact_if_spam(&mut spam_text).unwrap()); + assert!(process_text_for_spam_link(&mut spam_text, true).unwrap()); let url_redacted = "URL redacted for user protection"; assert_eq!(url_redacted, spam_text.unwrap()); let mut spam_text = Some(r"C:\Users\path\".to_string()); - assert!(check_and_redact_if_spam(&mut spam_text).unwrap()); + assert!(process_text_for_spam_link(&mut spam_text, true).unwrap()); let url_redacted = "URL redacted for user protection"; assert_eq!(url_redacted, spam_text.unwrap()); let mut valid_text = Some("Hello my name is NFT (The best ever!)".to_string()); - assert!(!check_and_redact_if_spam(&mut valid_text).unwrap()); + assert!(!process_text_for_spam_link(&mut valid_text, true).unwrap()); assert_eq!("Hello my name is NFT (The best ever!)", valid_text.unwrap()); let mut nft = nft(); - assert!(check_nft_metadata_for_spam(&mut nft).unwrap()); + assert!(process_metadata_for_spam_link(&mut nft, true).unwrap()); let meta_redacted = "{\"name\":\"URL redacted for user protection\",\"image\":\"https://tikimetadata.s3.amazonaws.com/tiki_box.png\"}"; assert_eq!(meta_redacted, nft.common.metadata.unwrap()) } diff --git a/mm2src/coins/nft/storage/db_test_helpers.rs b/mm2src/coins/nft/storage/db_test_helpers.rs index 583a7cc64c..d4534a4df7 100644 --- a/mm2src/coins/nft/storage/db_test_helpers.rs +++ b/mm2src/coins/nft/storage/db_test_helpers.rs @@ -243,7 +243,7 @@ fn nft_transfer_history() -> Vec { amount: BigDecimal::from_str("1").unwrap(), verified: Some(1), operator: Some("0x4ff0bbc9b64d635a4696d1a38554fb2529c103ff".to_string()), - possible_spam: true, + possible_spam: false, }, chain: Chain::Bsc, block_number: 25919780, @@ -274,7 +274,7 @@ fn nft_transfer_history() -> Vec { amount: BigDecimal::from_str("1").unwrap(), verified: Some(1), operator: None, - possible_spam: false, + possible_spam: true, }, chain: Chain::Bsc, block_number: 28056726, @@ -711,7 +711,7 @@ pub(crate) async fn test_get_update_transfer_meta_impl() { token_name: Some("Tiki box".to_string()), }; storage - .update_transfers_meta_by_token_addr_id(&chain, transfer_meta) + .update_transfers_meta_by_token_addr_id(&chain, transfer_meta, true) .await .unwrap(); let transfer_upd = storage @@ -720,6 +720,7 @@ pub(crate) async fn test_get_update_transfer_meta_impl() { .unwrap(); let transfer_upd = transfer_upd.get(0).unwrap(); assert_eq!(transfer_upd.token_name, Some("Tiki box".to_string())); + assert!(transfer_upd.common.possible_spam); } pub(crate) async fn test_update_transfer_spam_by_token_address_impl() { diff --git a/mm2src/coins/nft/storage/mod.rs b/mm2src/coins/nft/storage/mod.rs index 001d7485eb..14cc9243f0 100644 --- a/mm2src/coins/nft/storage/mod.rs +++ b/mm2src/coins/nft/storage/mod.rs @@ -164,10 +164,13 @@ pub trait NftTransferHistoryStorageOps { log_index: u32, ) -> MmResult, Self::Error>; + /// Updates the metadata for NFT transfers identified by their token address and ID. + /// Flags the transfers as `possible_spam` if `set_spam` is true. async fn update_transfers_meta_by_token_addr_id( &self, chain: &Chain, transfer_meta: TransferMeta, + set_spam: bool, ) -> MmResult<(), Self::Error>; async fn get_transfers_with_empty_meta(&self, chain: Chain) -> MmResult, Self::Error>; diff --git a/mm2src/coins/nft/storage/sql_storage.rs b/mm2src/coins/nft/storage/sql_storage.rs index 794b7c58ac..4e76b93249 100644 --- a/mm2src/coins/nft/storage/sql_storage.rs +++ b/mm2src/coins/nft/storage/sql_storage.rs @@ -467,6 +467,15 @@ fn update_transfers_meta_by_token_addr_id_sql(chain: &Chain) -> MmResult MmResult { + let table_name = chain.transfer_history_table_name()?; + let sql = format!( + "UPDATE {} SET possible_spam = ?1 WHERE token_address = ?2 AND token_id = ?3;", + table_name + ); + Ok(sql) +} + fn select_last_block_number_sql(table_name: String) -> MmResult { let sql = format!( "SELECT block_number FROM {} ORDER BY block_number DESC LIMIT 1", @@ -1121,6 +1130,7 @@ impl NftTransferHistoryStorageOps for SqliteNftStorage { &self, chain: &Chain, transfer_meta: TransferMeta, + set_spam: bool, ) -> MmResult<(), Self::Error> { let sql = update_transfers_meta_by_token_addr_id_sql(chain)?; let params = [ @@ -1130,6 +1140,12 @@ impl NftTransferHistoryStorageOps for SqliteNftStorage { transfer_meta.image_url, transfer_meta.image_domain, transfer_meta.token_name, + Some(transfer_meta.token_address.clone()), + Some(transfer_meta.token_id.to_string()), + ]; + let sql_spam = update_transfer_spam_by_token_addr_id(chain)?; + let params_spam = [ + Some(i32::from(true).to_string()), Some(transfer_meta.token_address), Some(transfer_meta.token_id.to_string()), ]; @@ -1138,6 +1154,9 @@ impl NftTransferHistoryStorageOps for SqliteNftStorage { let mut conn = selfi.0.lock().unwrap(); let sql_transaction = conn.transaction()?; sql_transaction.execute(&sql, params)?; + if set_spam { + sql_transaction.execute(&sql_spam, params_spam)?; + } sql_transaction.commit()?; Ok(()) }) diff --git a/mm2src/coins/nft/storage/wasm/wasm_storage.rs b/mm2src/coins/nft/storage/wasm/wasm_storage.rs index b6fd51765c..8234a6e296 100644 --- a/mm2src/coins/nft/storage/wasm/wasm_storage.rs +++ b/mm2src/coins/nft/storage/wasm/wasm_storage.rs @@ -653,6 +653,7 @@ impl NftTransferHistoryStorageOps for IndexedDbNftStorage { &self, chain: &Chain, transfer_meta: TransferMeta, + set_spam: bool, ) -> MmResult<(), Self::Error> { let locked_db = self.lock_db().await?; let db_transaction = locked_db.get_inner().transaction().await?; @@ -679,6 +680,9 @@ impl NftTransferHistoryStorageOps for IndexedDbNftStorage { transfer.image_url = transfer_meta.image_url.clone(); transfer.image_domain = transfer_meta.image_domain.clone(); transfer.token_name = transfer_meta.token_name.clone(); + if set_spam { + transfer.common.possible_spam = true; + } drop_mutability!(transfer); let index_keys = MultiIndex::new(NftTransferHistoryTable::CHAIN_TX_HASH_LOG_INDEX_INDEX) From 935d375c38a1d5a879f47483d2cb24166d27a90b Mon Sep 17 00:00:00 2001 From: laruh Date: Thu, 21 Sep 2023 18:21:11 +0700 Subject: [PATCH 26/39] add indexes for NftListTable --- mm2src/coins/nft/storage/wasm/wasm_storage.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mm2src/coins/nft/storage/wasm/wasm_storage.rs b/mm2src/coins/nft/storage/wasm/wasm_storage.rs index 8234a6e296..29da6a433c 100644 --- a/mm2src/coins/nft/storage/wasm/wasm_storage.rs +++ b/mm2src/coins/nft/storage/wasm/wasm_storage.rs @@ -970,6 +970,12 @@ impl TableSignature for NftListTable { table.create_multi_index(CHAIN_TOKEN_ADD_INDEX, &["chain", "token_address"], false)?; table.create_multi_index(CHAIN_TOKEN_DOMAIN_INDEX, &["chain", "token_domain"], false)?; table.create_multi_index(CHAIN_IMAGE_DOMAIN_INDEX, &["chain", "image_domain"], false)?; + table.create_multi_index( + Self::CHAIN_ANIMATION_DOMAIN_INDEX, + &["chain", "animation_domain"], + false, + )?; + table.create_multi_index(Self::CHAIN_EXTERNAL_DOMAIN_INDEX, &["chain", "external_domain"], false)?; table.create_index("chain", false)?; table.create_index("block_number", false)?; } From 6f33a11dc4a8cdb99f0d33ca39f2c9d5ee3deceb Mon Sep 17 00:00:00 2001 From: laruh Date: Fri, 22 Sep 2023 11:30:42 +0700 Subject: [PATCH 27/39] move `cannot_be_a_base` at the beginning of construct camo --- mm2src/coins/nft.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index 2dbb7ac96d..236c843d5d 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -781,13 +781,14 @@ async fn get_uri_meta(token_uri: Option<&str>, metadata: Option<&str>, url_antis } fn construct_camo_url_with_token(token_uri: &str, url_antispam: &Url) -> Option { + if url_antispam.cannot_be_a_base() { + return None; + } let mut url = url_antispam.clone(); url.set_path("url/decode"); let hex_token_uri = hex::encode(token_uri); if let Ok(mut segments) = url.path_segments_mut() { - segments.push(hex_token_uri.as_str()); - } else { - return None; + segments.push(&hex_token_uri); } Some(url) } From f35e9b6e91d0f8f87dd8e753f7c2fd53b0c6983a Mon Sep 17 00:00:00 2001 From: laruh Date: Fri, 22 Sep 2023 12:47:22 +0700 Subject: [PATCH 28/39] remove repetitive code in wasm update phishing --- mm2src/coins/nft/storage/wasm/wasm_storage.rs | 156 +++++++----------- 1 file changed, 58 insertions(+), 98 deletions(-) diff --git a/mm2src/coins/nft/storage/wasm/wasm_storage.rs b/mm2src/coins/nft/storage/wasm/wasm_storage.rs index 29da6a433c..789ec069da 100644 --- a/mm2src/coins/nft/storage/wasm/wasm_storage.rs +++ b/mm2src/coins/nft/storage/wasm/wasm_storage.rs @@ -456,69 +456,12 @@ impl NftListStorageOps for IndexedDbNftStorage { let table = db_transaction.table::().await?; let chain_str = chain.to_string(); - let index_keys = MultiIndex::new(CHAIN_TOKEN_DOMAIN_INDEX) - .with_value(&chain_str)? - .with_value(&domain)?; - let nfts_table = table.get_items_by_multi_index(index_keys).await?; - for (_item_id, item) in nfts_table.into_iter() { - let mut nft = nft_details_from_item(item)?; - nft.possible_phishing = possible_phishing; - drop_mutability!(nft); - let nft_item = NftListTable::from_nft(&nft)?; - let index_keys = MultiIndex::new(CHAIN_TOKEN_ADD_TOKEN_ID_INDEX) - .with_value(&chain_str)? - .with_value(eth_addr_to_hex(&nft.common.token_address))? - .with_value(nft.common.token_id.to_string())?; - table.replace_item_by_unique_multi_index(index_keys, &nft_item).await?; - } - - let index_keys = MultiIndex::new(CHAIN_IMAGE_DOMAIN_INDEX) - .with_value(&chain_str)? - .with_value(&domain)?; - let nfts_table = table.get_items_by_multi_index(index_keys).await?; - for (_item_id, item) in nfts_table.into_iter() { - let mut nft = nft_details_from_item(item)?; - nft.possible_phishing = possible_phishing; - drop_mutability!(nft); - let nft_item = NftListTable::from_nft(&nft)?; - let index_keys = MultiIndex::new(CHAIN_TOKEN_ADD_TOKEN_ID_INDEX) - .with_value(&chain_str)? - .with_value(eth_addr_to_hex(&nft.common.token_address))? - .with_value(nft.common.token_id.to_string())?; - table.replace_item_by_unique_multi_index(index_keys, &nft_item).await?; - } - - let index_keys = MultiIndex::new(NftListTable::CHAIN_ANIMATION_DOMAIN_INDEX) - .with_value(&chain_str)? - .with_value(&domain)?; - let nfts_table = table.get_items_by_multi_index(index_keys).await?; - for (_item_id, item) in nfts_table.into_iter() { - let mut nft = nft_details_from_item(item)?; - nft.possible_phishing = possible_phishing; - drop_mutability!(nft); - let nft_item = NftListTable::from_nft(&nft)?; - let index_keys = MultiIndex::new(CHAIN_TOKEN_ADD_TOKEN_ID_INDEX) - .with_value(&chain_str)? - .with_value(eth_addr_to_hex(&nft.common.token_address))? - .with_value(nft.common.token_id.to_string())?; - table.replace_item_by_unique_multi_index(index_keys, &nft_item).await?; - } - - let index_keys = MultiIndex::new(NftListTable::CHAIN_EXTERNAL_DOMAIN_INDEX) - .with_value(&chain_str)? - .with_value(&domain)?; - let nfts_table = table.get_items_by_multi_index(index_keys).await?; - for (_item_id, item) in nfts_table.into_iter() { - let mut nft = nft_details_from_item(item)?; - nft.possible_phishing = possible_phishing; - drop_mutability!(nft); - let nft_item = NftListTable::from_nft(&nft)?; - let index_keys = MultiIndex::new(CHAIN_TOKEN_ADD_TOKEN_ID_INDEX) - .with_value(&chain_str)? - .with_value(eth_addr_to_hex(&nft.common.token_address))? - .with_value(nft.common.token_id.to_string())?; - table.replace_item_by_unique_multi_index(index_keys, &nft_item).await?; - } + update_nft_phishing_for_index(&table, &chain_str, CHAIN_TOKEN_DOMAIN_INDEX, &domain, possible_phishing).await?; + update_nft_phishing_for_index(&table, &chain_str, CHAIN_IMAGE_DOMAIN_INDEX, &domain, possible_phishing).await?; + let animation_index = NftListTable::CHAIN_ANIMATION_DOMAIN_INDEX; + update_nft_phishing_for_index(&table, &chain_str, animation_index, &domain, possible_phishing).await?; + let external_index = NftListTable::CHAIN_EXTERNAL_DOMAIN_INDEX; + update_nft_phishing_for_index(&table, &chain_str, external_index, &domain, possible_phishing).await?; Ok(()) } } @@ -829,45 +772,62 @@ impl NftTransferHistoryStorageOps for IndexedDbNftStorage { let table = db_transaction.table::().await?; let chain_str = chain.to_string(); - let index_keys = MultiIndex::new(CHAIN_TOKEN_DOMAIN_INDEX) - .with_value(&chain_str)? - .with_value(&domain)?; - let transfers_table = table.get_items_by_multi_index(index_keys).await?; - for (_item_id, item) in transfers_table.into_iter() { - let mut transfer = transfer_details_from_item(item)?; - transfer.possible_phishing = possible_phishing; - drop_mutability!(transfer); - let transfer_item = NftTransferHistoryTable::from_transfer_history(&transfer)?; - let index_keys = MultiIndex::new(NftTransferHistoryTable::CHAIN_TX_HASH_LOG_INDEX_INDEX) - .with_value(&chain_str)? - .with_value(&transfer.common.transaction_hash)? - .with_value(transfer.common.log_index)?; - table - .replace_item_by_unique_multi_index(index_keys, &transfer_item) - .await?; - } - - let index_keys = MultiIndex::new(CHAIN_IMAGE_DOMAIN_INDEX) - .with_value(&chain_str)? - .with_value(&domain)?; - let transfers_table = table.get_items_by_multi_index(index_keys).await?; - for (_item_id, item) in transfers_table.into_iter() { - let mut transfer = transfer_details_from_item(item)?; - transfer.possible_phishing = possible_phishing; - drop_mutability!(transfer); - let transfer_item = NftTransferHistoryTable::from_transfer_history(&transfer)?; - let index_keys = MultiIndex::new(NftTransferHistoryTable::CHAIN_TX_HASH_LOG_INDEX_INDEX) - .with_value(&chain_str)? - .with_value(&transfer.common.transaction_hash)? - .with_value(transfer.common.log_index)?; - table - .replace_item_by_unique_multi_index(index_keys, &transfer_item) - .await?; - } + update_transfer_phishing_for_index(&table, &chain_str, CHAIN_TOKEN_DOMAIN_INDEX, &domain, possible_phishing) + .await?; + update_transfer_phishing_for_index(&table, &chain_str, CHAIN_IMAGE_DOMAIN_INDEX, &domain, possible_phishing) + .await?; Ok(()) } } +async fn update_transfer_phishing_for_index( + table: &DbTable<'_, NftTransferHistoryTable>, + chain: &str, + index: &str, + domain: &str, + possible_phishing: bool, +) -> MmResult<(), WasmNftCacheError> { + let index_keys = MultiIndex::new(index).with_value(chain)?.with_value(domain)?; + let transfers_table = table.get_items_by_multi_index(index_keys).await?; + for (_item_id, item) in transfers_table.into_iter() { + let mut transfer = transfer_details_from_item(item)?; + transfer.possible_phishing = possible_phishing; + drop_mutability!(transfer); + let transfer_item = NftTransferHistoryTable::from_transfer_history(&transfer)?; + let index_keys = MultiIndex::new(NftTransferHistoryTable::CHAIN_TX_HASH_LOG_INDEX_INDEX) + .with_value(chain)? + .with_value(&transfer.common.transaction_hash)? + .with_value(transfer.common.log_index)?; + table + .replace_item_by_unique_multi_index(index_keys, &transfer_item) + .await?; + } + Ok(()) +} + +async fn update_nft_phishing_for_index( + table: &DbTable<'_, NftListTable>, + chain: &str, + index: &str, + domain: &str, + possible_phishing: bool, +) -> MmResult<(), WasmNftCacheError> { + let index_keys = MultiIndex::new(index).with_value(chain)?.with_value(domain)?; + let nfts_table = table.get_items_by_multi_index(index_keys).await?; + for (_item_id, item) in nfts_table.into_iter() { + let mut nft = nft_details_from_item(item)?; + nft.possible_phishing = possible_phishing; + drop_mutability!(nft); + let nft_item = NftListTable::from_nft(&nft)?; + let index_keys = MultiIndex::new(CHAIN_TOKEN_ADD_TOKEN_ID_INDEX) + .with_value(chain)? + .with_value(eth_addr_to_hex(&nft.common.token_address))? + .with_value(nft.common.token_id.to_string())?; + table.replace_item_by_unique_multi_index(index_keys, &nft_item).await?; + } + Ok(()) +} + /// `get_last_block_from_table` function returns the highest block in the table related to certain blockchain type. async fn get_last_block_from_table( chain: &Chain, From 9abc1e850432cd9bf67538e5320589198b408a6b Mon Sep 17 00:00:00 2001 From: laruh Date: Fri, 22 Sep 2023 13:06:09 +0700 Subject: [PATCH 29/39] use match url.path_segments_mut() --- mm2src/coins/nft.rs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index 236c843d5d..e3e362131c 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -781,15 +781,13 @@ async fn get_uri_meta(token_uri: Option<&str>, metadata: Option<&str>, url_antis } fn construct_camo_url_with_token(token_uri: &str, url_antispam: &Url) -> Option { - if url_antispam.cannot_be_a_base() { - return None; - } let mut url = url_antispam.clone(); url.set_path("url/decode"); let hex_token_uri = hex::encode(token_uri); - if let Ok(mut segments) = url.path_segments_mut() { - segments.push(&hex_token_uri); - } + match url.path_segments_mut() { + Ok(mut segments) => segments.push(hex_token_uri.as_str()), + Err(_) => return None, + }; Some(url) } From 42719f36767b2ffa83b2efd5e98df37df0bc0026 Mon Sep 17 00:00:00 2001 From: laruh Date: Fri, 22 Sep 2023 16:58:06 +0700 Subject: [PATCH 30/39] move `hex_token_uri` --- mm2src/coins/nft.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index e3e362131c..24faf81cb0 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -783,9 +783,8 @@ async fn get_uri_meta(token_uri: Option<&str>, metadata: Option<&str>, url_antis fn construct_camo_url_with_token(token_uri: &str, url_antispam: &Url) -> Option { let mut url = url_antispam.clone(); url.set_path("url/decode"); - let hex_token_uri = hex::encode(token_uri); match url.path_segments_mut() { - Ok(mut segments) => segments.push(hex_token_uri.as_str()), + Ok(mut segments) => segments.push(hex::encode(token_uri).as_str()), Err(_) => return None, }; Some(url) From a7185eb737830b4722eba4e6681f1aacf2e926f8 Mon Sep 17 00:00:00 2001 From: laruh Date: Sat, 23 Sep 2023 18:45:22 +0700 Subject: [PATCH 31/39] add cross tests --- mm2src/coins/nft/nft_tests.rs | 626 ++++++++++++++++++++ mm2src/coins/nft/storage/db_test_helpers.rs | 14 +- 2 files changed, 630 insertions(+), 10 deletions(-) diff --git a/mm2src/coins/nft/nft_tests.rs b/mm2src/coins/nft/nft_tests.rs index be00a4d47c..bb5072446d 100644 --- a/mm2src/coins/nft/nft_tests.rs +++ b/mm2src/coins/nft/nft_tests.rs @@ -395,3 +395,629 @@ mod wasm_tests { #[wasm_bindgen_test] async fn test_update_transfer_phishing_by_domain() { test_update_transfer_phishing_by_domain_impl().await } } + +#[cfg(any(test, target_arch = "wasm32"))] +mod nft_cross_tests { + use crate::eth::eth_addr_to_hex; + use crate::nft::nft_structs::{Chain, MnemonicHQRes, NftFromMoralis, NftListFilters, NftTransferHistoryFilters, + NftTransferHistoryFromMoralis, PhishingDomainReq, PhishingDomainRes, + SpamContractReq, SpamContractRes, TransferMeta, UriMeta}; + use crate::nft::storage::db_test_helpers::{init_nft_history_storage, init_nft_list_storage, nft, nft_list, + nft_transfer_history}; + use crate::nft::storage::{NftListStorageOps, NftTransferHistoryStorageOps, RemoveNftResult}; + use crate::nft::{check_moralis_ipfs_bafy, get_domain_from_url, process_metadata_for_spam_link, + process_text_for_spam_link}; + use ethereum_types::Address; + use mm2_net::transport::send_post_request_to_uri; + use mm2_number::BigDecimal; + use std::num::NonZeroUsize; + use std::str::FromStr; + + const MORALIS_API_ENDPOINT_TEST: &str = "https://moralis-proxy.komodo.earth/api/v2"; + const TEST_WALLET_ADDR_EVM: &str = "0x394d86994f954ed931b86791b62fe64f4c5dac37"; + const BLOCKLIST_API_ENDPOINT: &str = "https://nft.antispam.dragonhound.info"; + const TOKEN_ADD: &str = "0xfd913a305d70a60aac4faac70c739563738e1f81"; + const TOKEN_ID: &str = "214300044414"; + const TX_HASH: &str = "0x1e9f04e9b571b283bde02c98c2a97da39b2bb665b57c1f2b0b733f9b681debbe"; + const LOG_INDEX: u32 = 495; + + #[cfg(not(target_arch = "wasm32"))] + use mm2_net::native_http::send_request_to_uri; + + common::cfg_wasm32! { + use wasm_bindgen_test::*; + wasm_bindgen_test::wasm_bindgen_test_configure!(run_in_browser); + use mm2_net::wasm_http::send_request_to_uri; + } + + macro_rules! cross_test { + ($test_name:ident, $test_code:block) => { + #[cfg(not(target_arch = "wasm32"))] + #[tokio::test(flavor = "multi_thread")] + async fn $test_name() { $test_code } + + #[cfg(target_arch = "wasm32")] + #[wasm_bindgen_test] + async fn $test_name() { $test_code } + }; + } + + cross_test!(test_moralis_ipfs_bafy, { + let uri = + "https://ipfs.moralis.io:2053/ipfs/bafybeifnek24coy5xj5qabdwh24dlp5omq34nzgvazkfyxgnqms4eidsiq/1.json"; + let res_uri = check_moralis_ipfs_bafy(Some(uri)); + let expected = "https://ipfs.io/ipfs/bafybeifnek24coy5xj5qabdwh24dlp5omq34nzgvazkfyxgnqms4eidsiq/1.json"; + assert_eq!(expected, res_uri.unwrap()); + }); + + cross_test!(test_get_domain_from_url, { + let image_url = "https://public.nftstatic.com/static/nft/res/4df0a5da04174e1e9be04b22a805f605.png"; + let res_domain = get_domain_from_url(Some(image_url)); + let expected = "public.nftstatic.com"; + assert_eq!(expected, res_domain.unwrap()); + }); + + cross_test!(test_invalid_moralis_ipfs_link, { + let uri = "example.com/bafy?1=ipfs.moralis.io&e=https://"; + let res_uri = check_moralis_ipfs_bafy(Some(uri)); + assert_eq!(uri, res_uri.unwrap()); + }); + + cross_test!(test_check_for_spam_links, { + let mut spam_text = Some("https://arweave.net".to_string()); + assert!(process_text_for_spam_link(&mut spam_text, true).unwrap()); + let url_redacted = "URL redacted for user protection"; + assert_eq!(url_redacted, spam_text.unwrap()); + + let mut spam_text = Some("ftp://123path ".to_string()); + assert!(process_text_for_spam_link(&mut spam_text, true).unwrap()); + let url_redacted = "URL redacted for user protection"; + assert_eq!(url_redacted, spam_text.unwrap()); + + let mut spam_text = Some("/192.168.1.1/some.example.org?type=A".to_string()); + assert!(process_text_for_spam_link(&mut spam_text, true).unwrap()); + let url_redacted = "URL redacted for user protection"; + assert_eq!(url_redacted, spam_text.unwrap()); + + let mut spam_text = Some(r"C:\Users\path\".to_string()); + assert!(process_text_for_spam_link(&mut spam_text, true).unwrap()); + let url_redacted = "URL redacted for user protection"; + assert_eq!(url_redacted, spam_text.unwrap()); + + let mut valid_text = Some("Hello my name is NFT (The best ever!)".to_string()); + assert!(!process_text_for_spam_link(&mut valid_text, true).unwrap()); + assert_eq!("Hello my name is NFT (The best ever!)", valid_text.unwrap()); + + let mut nft = nft(); + assert!(process_metadata_for_spam_link(&mut nft, true).unwrap()); + let meta_redacted = "{\"name\":\"URL redacted for user protection\",\"image\":\"https://tikimetadata.s3.amazonaws.com/tiki_box.png\"}"; + assert_eq!(meta_redacted, nft.common.metadata.unwrap()) + }); + + cross_test!(test_moralis_requests, { + let uri_nft_list = format!( + "{}/{}/nft?chain=POLYGON&format=decimal", + MORALIS_API_ENDPOINT_TEST, TEST_WALLET_ADDR_EVM + ); + let response_nft_list = send_request_to_uri(uri_nft_list.as_str()).await.unwrap(); + let nfts_list = response_nft_list["result"].as_array().unwrap(); + for nft_json in nfts_list { + let nft_moralis: NftFromMoralis = serde_json::from_str(&nft_json.to_string()).unwrap(); + assert_eq!(TEST_WALLET_ADDR_EVM, eth_addr_to_hex(&nft_moralis.common.owner_of)); + } + + let uri_history = format!( + "{}/{}/nft/transfers?chain=POLYGON&format=decimal", + MORALIS_API_ENDPOINT_TEST, TEST_WALLET_ADDR_EVM + ); + let response_transfer_history = send_request_to_uri(uri_history.as_str()).await.unwrap(); + let mut transfer_list = response_transfer_history["result"].as_array().unwrap().clone(); + assert!(!transfer_list.is_empty()); + let first_transfer = transfer_list.remove(transfer_list.len() - 1); + let transfer_moralis: NftTransferHistoryFromMoralis = + serde_json::from_str(&first_transfer.to_string()).unwrap(); + assert_eq!( + TEST_WALLET_ADDR_EVM, + eth_addr_to_hex(&transfer_moralis.common.to_address) + ); + + let uri_meta = format!( + "{}/nft/0xed55e4477b795eaa9bb4bca24df42214e1a05c18/1111777?chain=POLYGON&format=decimal", + MORALIS_API_ENDPOINT_TEST + ); + let response_meta = send_request_to_uri(uri_meta.as_str()).await.unwrap(); + let nft_moralis: NftFromMoralis = serde_json::from_str(&response_meta.to_string()).unwrap(); + assert_eq!(41237364, *nft_moralis.block_number_minted.unwrap()); + }); + + cross_test!(test_antispam_wallet_endpoint, { + let uri_mnemonichq = format!( + "{}/api/blocklist/wallet/eth/0x3eb4b12127EdC81A4d2fD49658db07005bcAd065", + BLOCKLIST_API_ENDPOINT + ); + let res_value = send_request_to_uri(uri_mnemonichq.as_str()).await.unwrap(); + let mnemonichq_res: MnemonicHQRes = serde_json::from_value(res_value).unwrap(); + assert!(mnemonichq_res + .spam_contracts + .contains(&Address::from_str("0x0ded8542fc8b2b4e781b96e99fee6406550c9b7c").unwrap())); + assert_eq!(Chain::Eth, mnemonichq_res.network); + }); + + cross_test!(test_antispam_scan_endpoints, { + let req_spam = SpamContractReq { + network: Chain::Eth, + addresses: "0x0ded8542fc8b2b4e781b96e99fee6406550c9b7c,0x8d1355b65da254f2cc4611453adfa8b7a13f60ee" + .to_string(), + }; + let uri_contract = format!("{}/api/blocklist/contract/scan", BLOCKLIST_API_ENDPOINT); + let req_json = serde_json::to_string(&req_spam).unwrap(); + let contract_scan_res = send_post_request_to_uri(uri_contract.as_str(), req_json).await.unwrap(); + let spam_res: SpamContractRes = serde_json::from_slice(&contract_scan_res).unwrap(); + assert!(spam_res + .result + .get(&Address::from_str("0x0ded8542fc8b2b4e781b96e99fee6406550c9b7c").unwrap()) + .unwrap()); + assert!(spam_res + .result + .get(&Address::from_str("0x8d1355b65da254f2cc4611453adfa8b7a13f60ee").unwrap()) + .unwrap()); + + let req_phishing = PhishingDomainReq { + domains: "disposal-account-case-1f677.web.app,defi8090.vip".to_string(), + }; + let req_json = serde_json::to_string(&req_phishing).unwrap(); + let uri_domain = format!("{}/api/blocklist/domain/scan", BLOCKLIST_API_ENDPOINT); + let domain_scan_res = send_post_request_to_uri(uri_domain.as_str(), req_json).await.unwrap(); + let phishing_res: PhishingDomainRes = serde_json::from_slice(&domain_scan_res).unwrap(); + assert!(phishing_res.result.get("disposal-account-case-1f677.web.app").unwrap()); + }); + + cross_test!(test_camo, { + let hex_token_uri = hex::encode("https://tikimetadata.s3.amazonaws.com/tiki_box.json"); + let uri_decode = format!("{}/url/decode/{}", BLOCKLIST_API_ENDPOINT, hex_token_uri); + let decode_res = send_request_to_uri(&uri_decode).await.unwrap(); + let uri_meta: UriMeta = serde_json::from_value(decode_res).unwrap(); + assert_eq!( + uri_meta.raw_image_url.unwrap(), + "https://tikimetadata.s3.amazonaws.com/tiki_box.png" + ); + }); + + ///// + + cross_test!(test_add_get_nfts, { + let chain = Chain::Bsc; + let storage = init_nft_list_storage(&chain).await; + let nft_list = nft_list(); + storage.add_nfts_to_list(chain, nft_list, 28056726).await.unwrap(); + + let token_id = BigDecimal::from_str(TOKEN_ID).unwrap(); + let nft = storage + .get_nft(&chain, TOKEN_ADD.to_string(), token_id) + .await + .unwrap() + .unwrap(); + assert_eq!(nft.block_number, 28056721); + }); + + cross_test!(test_last_nft_block, { + let chain = Chain::Bsc; + let storage = init_nft_list_storage(&chain).await; + let nft_list = nft_list(); + storage.add_nfts_to_list(chain, nft_list, 28056726).await.unwrap(); + + let last_block = NftListStorageOps::get_last_block_number(&storage, &chain) + .await + .unwrap() + .unwrap(); + assert_eq!(last_block, 28056726); + }); + + cross_test!(test_nft_list, { + let chain = Chain::Bsc; + let storage = init_nft_list_storage(&chain).await; + let nft_list = nft_list(); + storage.add_nfts_to_list(chain, nft_list, 28056726).await.unwrap(); + + let nft_list = storage + .get_nft_list(vec![chain], false, 1, Some(NonZeroUsize::new(3).unwrap()), None) + .await + .unwrap(); + assert_eq!(nft_list.nfts.len(), 1); + let nft = nft_list.nfts.get(0).unwrap(); + assert_eq!(nft.block_number, 28056721); + assert_eq!(nft_list.skipped, 2); + assert_eq!(nft_list.total, 4); + }); + + cross_test!(test_remove_nft, { + let chain = Chain::Bsc; + let storage = init_nft_list_storage(&chain).await; + let nft_list = nft_list(); + storage.add_nfts_to_list(chain, nft_list, 28056726).await.unwrap(); + + let token_id = BigDecimal::from_str(TOKEN_ID).unwrap(); + let remove_rslt = storage + .remove_nft_from_list(&chain, TOKEN_ADD.to_string(), token_id, 28056800) + .await + .unwrap(); + assert_eq!(remove_rslt, RemoveNftResult::NftRemoved); + let list_len = storage + .get_nft_list(vec![chain], true, 1, None, None) + .await + .unwrap() + .nfts + .len(); + assert_eq!(list_len, 3); + let last_scanned_block = storage.get_last_scanned_block(&chain).await.unwrap().unwrap(); + assert_eq!(last_scanned_block, 28056800); + }); + + cross_test!(test_nft_amount, { + let chain = Chain::Bsc; + let storage = init_nft_list_storage(&chain).await; + let mut nft = nft(); + storage + .add_nfts_to_list(chain, vec![nft.clone()], 25919780) + .await + .unwrap(); + + nft.common.amount -= BigDecimal::from(1); + storage.update_nft_amount(&chain, nft.clone(), 25919800).await.unwrap(); + let amount = storage + .get_nft_amount( + &chain, + eth_addr_to_hex(&nft.common.token_address), + nft.common.token_id.clone(), + ) + .await + .unwrap() + .unwrap(); + assert_eq!(amount, "1"); + let last_scanned_block = storage.get_last_scanned_block(&chain).await.unwrap().unwrap(); + assert_eq!(last_scanned_block, 25919800); + + nft.common.amount += BigDecimal::from(1); + nft.block_number = 25919900; + storage + .update_nft_amount_and_block_number(&chain, nft.clone()) + .await + .unwrap(); + let amount = storage + .get_nft_amount(&chain, eth_addr_to_hex(&nft.common.token_address), nft.common.token_id) + .await + .unwrap() + .unwrap(); + assert_eq!(amount, "2"); + let last_scanned_block = storage.get_last_scanned_block(&chain).await.unwrap().unwrap(); + assert_eq!(last_scanned_block, 25919900); + }); + + cross_test!(test_refresh_metadata, { + let chain = Chain::Bsc; + let storage = init_nft_list_storage(&chain).await; + let new_symbol = "NEW_SYMBOL"; + let mut nft = nft(); + storage + .add_nfts_to_list(chain, vec![nft.clone()], 25919780) + .await + .unwrap(); + nft.common.symbol = Some(new_symbol.to_string()); + drop_mutability!(nft); + let token_add = eth_addr_to_hex(&nft.common.token_address); + let token_id = nft.common.token_id.clone(); + storage.refresh_nft_metadata(&chain, nft).await.unwrap(); + let nft_upd = storage.get_nft(&chain, token_add, token_id).await.unwrap().unwrap(); + assert_eq!(new_symbol.to_string(), nft_upd.common.symbol.unwrap()); + }); + + cross_test!(test_update_nft_spam_by_token_address, { + let chain = Chain::Bsc; + let storage = init_nft_list_storage(&chain).await; + let nft_list = nft_list(); + storage.add_nfts_to_list(chain, nft_list, 28056726).await.unwrap(); + + storage + .update_nft_spam_by_token_address(&chain, TOKEN_ADD.to_string(), true) + .await + .unwrap(); + let nfts = storage + .get_nfts_by_token_address(chain, TOKEN_ADD.to_string()) + .await + .unwrap(); + for nft in nfts { + assert!(nft.common.possible_spam); + } + }); + + cross_test!(test_exclude_nft_spam, { + let chain = Chain::Bsc; + let storage = init_nft_list_storage(&chain).await; + let nft_list = nft_list(); + storage.add_nfts_to_list(chain, nft_list, 28056726).await.unwrap(); + + let filters = NftListFilters { + exclude_spam: true, + exclude_phishing: false, + }; + let nft_list = storage + .get_nft_list(vec![chain], true, 1, None, Some(filters)) + .await + .unwrap(); + assert_eq!(nft_list.nfts.len(), 3); + }); + + cross_test!(test_get_animation_external_domains, { + let chain = Chain::Bsc; + let storage = init_nft_list_storage(&chain).await; + let nft_list = nft_list(); + storage.add_nfts_to_list(chain, nft_list, 28056726).await.unwrap(); + + let domains = storage.get_animation_external_domains(&chain).await.unwrap(); + assert_eq!(2, domains.len()); + assert!(domains.contains("tikimetadata.s3.amazonaws.com")); + assert!(domains.contains("public.nftstatic.com")); + }); + + cross_test!(test_update_nft_phishing_by_domain, { + let chain = Chain::Bsc; + let storage = init_nft_list_storage(&chain).await; + let nft_list = nft_list(); + storage.add_nfts_to_list(chain, nft_list, 28056726).await.unwrap(); + + let domains = vec![ + "tikimetadata.s3.amazonaws.com".to_string(), + "public.nftstatic.com".to_string(), + ]; + for domain in domains.into_iter() { + storage + .update_nft_phishing_by_domain(&chain, domain, true) + .await + .unwrap(); + } + let nfts = storage + .get_nft_list(vec![chain], true, 1, None, None) + .await + .unwrap() + .nfts; + for nft in nfts.into_iter() { + assert!(nft.possible_phishing); + } + }); + + cross_test!(test_add_get_transfers, { + let chain = Chain::Bsc; + let storage = init_nft_history_storage(&chain).await; + let transfers = nft_transfer_history(); + storage.add_transfers_to_history(chain, transfers).await.unwrap(); + + let token_id = BigDecimal::from_str(TOKEN_ID).unwrap(); + let transfer1 = storage + .get_transfers_by_token_addr_id(chain, TOKEN_ADD.to_string(), token_id) + .await + .unwrap() + .get(0) + .unwrap() + .clone(); + assert_eq!(transfer1.block_number, 28056721); + let transfer2 = storage + .get_transfer_by_tx_hash_and_log_index(&chain, TX_HASH.to_string(), LOG_INDEX) + .await + .unwrap() + .unwrap(); + assert_eq!(transfer2.block_number, 28056726); + let transfer_from = storage.get_transfers_from_block(chain, 28056721).await.unwrap(); + assert_eq!(transfer_from.len(), 3); + }); + + cross_test!(test_last_transfer_block, { + let chain = Chain::Bsc; + let storage = init_nft_history_storage(&chain).await; + let transfers = nft_transfer_history(); + storage.add_transfers_to_history(chain, transfers).await.unwrap(); + + let last_block = NftTransferHistoryStorageOps::get_last_block_number(&storage, &chain) + .await + .unwrap() + .unwrap(); + assert_eq!(last_block, 28056726); + }); + + cross_test!(test_transfer_history, { + let chain = Chain::Bsc; + let storage = init_nft_history_storage(&chain).await; + let transfers = nft_transfer_history(); + storage.add_transfers_to_history(chain, transfers).await.unwrap(); + + let transfer_history = storage + .get_transfer_history(vec![chain], false, 1, Some(NonZeroUsize::new(3).unwrap()), None) + .await + .unwrap(); + assert_eq!(transfer_history.transfer_history.len(), 1); + let transfer = transfer_history.transfer_history.get(0).unwrap(); + assert_eq!(transfer.block_number, 28056721); + assert_eq!(transfer_history.skipped, 2); + assert_eq!(transfer_history.total, 4); + }); + + cross_test!(test_transfer_history_filters, { + let chain = Chain::Bsc; + let storage = init_nft_history_storage(&chain).await; + let transfers = nft_transfer_history(); + storage.add_transfers_to_history(chain, transfers).await.unwrap(); + + let filters = NftTransferHistoryFilters { + receive: true, + send: false, + from_date: None, + to_date: None, + exclude_spam: false, + exclude_phishing: false, + }; + + let filters1 = NftTransferHistoryFilters { + receive: false, + send: false, + from_date: None, + to_date: Some(1677166110), + exclude_spam: false, + exclude_phishing: false, + }; + + let filters2 = NftTransferHistoryFilters { + receive: false, + send: false, + from_date: Some(1677166110), + to_date: Some(1683627417), + exclude_spam: false, + exclude_phishing: false, + }; + + let transfer_history = storage + .get_transfer_history(vec![chain], true, 1, None, Some(filters)) + .await + .unwrap(); + assert_eq!(transfer_history.transfer_history.len(), 4); + let transfer = transfer_history.transfer_history.get(0).unwrap(); + assert_eq!(transfer.block_number, 28056726); + + let transfer_history1 = storage + .get_transfer_history(vec![chain], true, 1, None, Some(filters1)) + .await + .unwrap(); + assert_eq!(transfer_history1.transfer_history.len(), 1); + let transfer1 = transfer_history1.transfer_history.get(0).unwrap(); + assert_eq!(transfer1.block_number, 25919780); + + let transfer_history2 = storage + .get_transfer_history(vec![chain], true, 1, None, Some(filters2)) + .await + .unwrap(); + assert_eq!(transfer_history2.transfer_history.len(), 2); + let transfer_0 = transfer_history2.transfer_history.get(0).unwrap(); + assert_eq!(transfer_0.block_number, 28056721); + let transfer_1 = transfer_history2.transfer_history.get(1).unwrap(); + assert_eq!(transfer_1.block_number, 25919780); + }); + + cross_test!(test_get_update_transfer_meta, { + let chain = Chain::Bsc; + let storage = init_nft_history_storage(&chain).await; + let transfers = nft_transfer_history(); + storage.add_transfers_to_history(chain, transfers).await.unwrap(); + + let vec_token_add_id = storage.get_transfers_with_empty_meta(chain).await.unwrap(); + assert_eq!(vec_token_add_id.len(), 3); + + let token_add = "0x5c7d6712dfaf0cb079d48981781c8705e8417ca0".to_string(); + let transfer_meta = TransferMeta { + token_address: token_add.clone(), + token_id: Default::default(), + token_uri: None, + token_domain: None, + collection_name: None, + image_url: None, + image_domain: None, + token_name: Some("Tiki box".to_string()), + }; + storage + .update_transfers_meta_by_token_addr_id(&chain, transfer_meta, true) + .await + .unwrap(); + let transfer_upd = storage + .get_transfers_by_token_addr_id(chain, token_add, Default::default()) + .await + .unwrap(); + let transfer_upd = transfer_upd.get(0).unwrap(); + assert_eq!(transfer_upd.token_name, Some("Tiki box".to_string())); + assert!(transfer_upd.common.possible_spam); + }); + + cross_test!(test_update_transfer_spam_by_token_address, { + let chain = Chain::Bsc; + let storage = init_nft_history_storage(&chain).await; + let transfers = nft_transfer_history(); + storage.add_transfers_to_history(chain, transfers).await.unwrap(); + + storage + .update_transfer_spam_by_token_address(&chain, TOKEN_ADD.to_string(), true) + .await + .unwrap(); + let transfers = storage + .get_transfers_by_token_address(chain, TOKEN_ADD.to_string()) + .await + .unwrap(); + for transfers in transfers { + assert!(transfers.common.possible_spam); + } + }); + + cross_test!(test_get_token_addresses, { + let chain = Chain::Bsc; + let storage = init_nft_history_storage(&chain).await; + let transfers = nft_transfer_history(); + storage.add_transfers_to_history(chain, transfers).await.unwrap(); + + let token_addresses = storage.get_token_addresses(chain).await.unwrap(); + assert_eq!(token_addresses.len(), 2); + }); + + cross_test!(test_exclude_transfer_spam, { + let chain = Chain::Bsc; + let storage = init_nft_history_storage(&chain).await; + let transfers = nft_transfer_history(); + storage.add_transfers_to_history(chain, transfers).await.unwrap(); + + let filters = NftTransferHistoryFilters { + receive: true, + send: true, + from_date: None, + to_date: None, + exclude_spam: true, + exclude_phishing: false, + }; + let transfer_history = storage + .get_transfer_history(vec![chain], true, 1, None, Some(filters)) + .await + .unwrap(); + assert_eq!(transfer_history.transfer_history.len(), 3); + }); + + cross_test!(test_get_domains, { + let chain = Chain::Bsc; + let storage = init_nft_history_storage(&chain).await; + let transfers = nft_transfer_history(); + storage.add_transfers_to_history(chain, transfers).await.unwrap(); + + let domains = storage.get_domains(&chain).await.unwrap(); + assert_eq!(2, domains.len()); + assert!(domains.contains("tikimetadata.s3.amazonaws.com")); + assert!(domains.contains("public.nftstatic.com")); + }); + + cross_test!(test_update_transfer_phishing_by_domain, { + let chain = Chain::Bsc; + let storage = init_nft_history_storage(&chain).await; + let transfers = nft_transfer_history(); + storage.add_transfers_to_history(chain, transfers).await.unwrap(); + + let domains = vec![ + "tikimetadata.s3.amazonaws.com".to_string(), + "public.nftstatic.com".to_string(), + ]; + for domain in domains.into_iter() { + storage + .update_transfer_phishing_by_domain(&chain, domain, true) + .await + .unwrap(); + } + let transfers = storage + .get_transfer_history(vec![chain], true, 1, None, None) + .await + .unwrap() + .transfer_history; + for transfer in transfers.into_iter() { + assert!(transfer.possible_phishing); + } + }); +} diff --git a/mm2src/coins/nft/storage/db_test_helpers.rs b/mm2src/coins/nft/storage/db_test_helpers.rs index d4534a4df7..69ecae1f76 100644 --- a/mm2src/coins/nft/storage/db_test_helpers.rs +++ b/mm2src/coins/nft/storage/db_test_helpers.rs @@ -8,12 +8,6 @@ use mm2_test_helpers::for_tests::mm_ctx_with_custom_db; use std::num::NonZeroUsize; use std::str::FromStr; -cfg_wasm32! { - use wasm_bindgen_test::*; - - wasm_bindgen_test_configure!(run_in_browser); -} - const TOKEN_ADD: &str = "0xfd913a305d70a60aac4faac70c739563738e1f81"; const TOKEN_ID: &str = "214300044414"; const TX_HASH: &str = "0x1e9f04e9b571b283bde02c98c2a97da39b2bb665b57c1f2b0b733f9b681debbe"; @@ -61,7 +55,7 @@ pub(crate) fn nft() -> Nft { } } -fn nft_list() -> Vec { +pub(crate) fn nft_list() -> Vec { let nft = Nft { common: NftCommon { token_address: Address::from_str("0x5c7d6712dfaf0cb079d48981781c8705e8417ca0").unwrap(), @@ -227,7 +221,7 @@ fn nft_list() -> Vec { vec![nft, nft1, nft2, nft3] } -fn nft_transfer_history() -> Vec { +pub(crate) fn nft_transfer_history() -> Vec { let transfer = NftTransferHistory { common: NftTransferCommon { block_hash: Some("0xcb41654fc5cf2bf5d7fd3f061693405c74d419def80993caded0551ecfaeaae5".to_string()), @@ -355,7 +349,7 @@ fn nft_transfer_history() -> Vec { vec![transfer, transfer1, transfer2, transfer3] } -async fn init_nft_list_storage(chain: &Chain) -> impl NftListStorageOps + NftTransferHistoryStorageOps { +pub(crate) async fn init_nft_list_storage(chain: &Chain) -> impl NftListStorageOps + NftTransferHistoryStorageOps { let ctx = mm_ctx_with_custom_db(); let storage = NftStorageBuilder::new(&ctx).build().unwrap(); NftListStorageOps::init(&storage, chain).await.unwrap(); @@ -364,7 +358,7 @@ async fn init_nft_list_storage(chain: &Chain) -> impl NftListStorageOps + NftTra storage } -async fn init_nft_history_storage(chain: &Chain) -> impl NftListStorageOps + NftTransferHistoryStorageOps { +pub(crate) async fn init_nft_history_storage(chain: &Chain) -> impl NftListStorageOps + NftTransferHistoryStorageOps { let ctx = mm_ctx_with_custom_db(); let storage = NftStorageBuilder::new(&ctx).build().unwrap(); NftTransferHistoryStorageOps::init(&storage, chain).await.unwrap(); From 840a3a183513b65e1d96a42e83f32274bbd9aac9 Mon Sep 17 00:00:00 2001 From: laruh Date: Sat, 23 Sep 2023 19:21:34 +0700 Subject: [PATCH 32/39] leave only cross tests --- mm2src/coins/nft/nft_tests.rs | 1602 +++++++------------ mm2src/coins/nft/storage/db_test_helpers.rs | 449 +----- 2 files changed, 601 insertions(+), 1450 deletions(-) diff --git a/mm2src/coins/nft/nft_tests.rs b/mm2src/coins/nft/nft_tests.rs index bb5072446d..a7dc008d0e 100644 --- a/mm2src/coins/nft/nft_tests.rs +++ b/mm2src/coins/nft/nft_tests.rs @@ -1,1023 +1,617 @@ +use crate::eth::eth_addr_to_hex; +use crate::nft::nft_structs::{Chain, MnemonicHQRes, NftFromMoralis, NftListFilters, NftTransferHistoryFilters, + NftTransferHistoryFromMoralis, PhishingDomainReq, PhishingDomainRes, SpamContractReq, + SpamContractRes, TransferMeta, UriMeta}; +use crate::nft::storage::db_test_helpers::{init_nft_history_storage, init_nft_list_storage, nft, nft_list, + nft_transfer_history}; +use crate::nft::storage::{NftListStorageOps, NftTransferHistoryStorageOps, RemoveNftResult}; +use crate::nft::{check_moralis_ipfs_bafy, get_domain_from_url, process_metadata_for_spam_link, + process_text_for_spam_link}; +use ethereum_types::Address; +use mm2_net::transport::send_post_request_to_uri; +use mm2_number::BigDecimal; +use std::num::NonZeroUsize; +use std::str::FromStr; + const MORALIS_API_ENDPOINT_TEST: &str = "https://moralis-proxy.komodo.earth/api/v2"; const TEST_WALLET_ADDR_EVM: &str = "0x394d86994f954ed931b86791b62fe64f4c5dac37"; const BLOCKLIST_API_ENDPOINT: &str = "https://nft.antispam.dragonhound.info"; +const TOKEN_ADD: &str = "0xfd913a305d70a60aac4faac70c739563738e1f81"; +const TOKEN_ID: &str = "214300044414"; +const TX_HASH: &str = "0x1e9f04e9b571b283bde02c98c2a97da39b2bb665b57c1f2b0b733f9b681debbe"; +const LOG_INDEX: u32 = 495; -#[cfg(all(test, not(target_arch = "wasm32")))] -mod native_tests { - use crate::eth::eth_addr_to_hex; - use crate::nft::nft_structs::{Chain, MnemonicHQRes, NftFromMoralis, NftTransferHistoryFromMoralis, - PhishingDomainReq, PhishingDomainRes, SpamContractReq, SpamContractRes, UriMeta}; - use crate::nft::nft_tests::{BLOCKLIST_API_ENDPOINT, MORALIS_API_ENDPOINT_TEST, TEST_WALLET_ADDR_EVM}; - use crate::nft::storage::db_test_helpers::*; - use crate::nft::{check_moralis_ipfs_bafy, get_domain_from_url, process_metadata_for_spam_link, - process_text_for_spam_link}; - use common::block_on; - use ethereum_types::Address; - use mm2_net::native_http::send_request_to_uri; - use mm2_net::transport::send_post_request_to_uri; - use std::str::FromStr; - - #[test] - fn test_moralis_ipfs_bafy() { - let uri = - "https://ipfs.moralis.io:2053/ipfs/bafybeifnek24coy5xj5qabdwh24dlp5omq34nzgvazkfyxgnqms4eidsiq/1.json"; - let res_uri = check_moralis_ipfs_bafy(Some(uri)); - let expected = "https://ipfs.io/ipfs/bafybeifnek24coy5xj5qabdwh24dlp5omq34nzgvazkfyxgnqms4eidsiq/1.json"; - assert_eq!(expected, res_uri.unwrap()); - } - - #[test] - fn test_get_domain_from_url() { - let image_url = "https://public.nftstatic.com/static/nft/res/4df0a5da04174e1e9be04b22a805f605.png"; - let res_domain = get_domain_from_url(Some(image_url)); - let expected = "public.nftstatic.com"; - assert_eq!(expected, res_domain.unwrap()); - } - - #[test] - fn test_invalid_moralis_ipfs_link() { - let uri = "example.com/bafy?1=ipfs.moralis.io&e=https://"; - let res_uri = check_moralis_ipfs_bafy(Some(uri)); - assert_eq!(uri, res_uri.unwrap()); - } - - #[test] - fn test_check_for_spam_links() { - let mut spam_text = Some("https://arweave.net".to_string()); - assert!(process_text_for_spam_link(&mut spam_text, true).unwrap()); - let url_redacted = "URL redacted for user protection"; - assert_eq!(url_redacted, spam_text.unwrap()); - - let mut spam_text = Some("ftp://123path ".to_string()); - assert!(process_text_for_spam_link(&mut spam_text, true).unwrap()); - let url_redacted = "URL redacted for user protection"; - assert_eq!(url_redacted, spam_text.unwrap()); - - let mut spam_text = Some("/192.168.1.1/some.example.org?type=A".to_string()); - assert!(process_text_for_spam_link(&mut spam_text, true).unwrap()); - let url_redacted = "URL redacted for user protection"; - assert_eq!(url_redacted, spam_text.unwrap()); - - let mut spam_text = Some(r"C:\Users\path\".to_string()); - assert!(process_text_for_spam_link(&mut spam_text, true).unwrap()); - let url_redacted = "URL redacted for user protection"; - assert_eq!(url_redacted, spam_text.unwrap()); - - let mut valid_text = Some("Hello my name is NFT (The best ever!)".to_string()); - assert!(!process_text_for_spam_link(&mut valid_text, true).unwrap()); - assert_eq!("Hello my name is NFT (The best ever!)", valid_text.unwrap()); - - let mut nft = nft(); - assert!(process_metadata_for_spam_link(&mut nft, true).unwrap()); - let meta_redacted = "{\"name\":\"URL redacted for user protection\",\"image\":\"https://tikimetadata.s3.amazonaws.com/tiki_box.png\"}"; - assert_eq!(meta_redacted, nft.common.metadata.unwrap()) - } - - #[test] - fn test_moralis_requests() { - let uri_nft_list = format!( - "{}/{}/nft?chain=POLYGON&format=decimal", - MORALIS_API_ENDPOINT_TEST, TEST_WALLET_ADDR_EVM - ); - let response_nft_list = block_on(send_request_to_uri(uri_nft_list.as_str())).unwrap(); - let nfts_list = response_nft_list["result"].as_array().unwrap(); - for nft_json in nfts_list { - let nft_moralis: NftFromMoralis = serde_json::from_str(&nft_json.to_string()).unwrap(); - assert_eq!(TEST_WALLET_ADDR_EVM, eth_addr_to_hex(&nft_moralis.common.owner_of)); - } - - let uri_history = format!( - "{}/{}/nft/transfers?chain=POLYGON&format=decimal", - MORALIS_API_ENDPOINT_TEST, TEST_WALLET_ADDR_EVM - ); - let response_transfer_history = block_on(send_request_to_uri(uri_history.as_str())).unwrap(); - let mut transfer_list = response_transfer_history["result"].as_array().unwrap().clone(); - assert!(!transfer_list.is_empty()); - let first_transfer = transfer_list.remove(transfer_list.len() - 1); - let transfer_moralis: NftTransferHistoryFromMoralis = - serde_json::from_str(&first_transfer.to_string()).unwrap(); - assert_eq!( - TEST_WALLET_ADDR_EVM, - eth_addr_to_hex(&transfer_moralis.common.to_address) - ); - - let uri_meta = format!( - "{}/nft/0xed55e4477b795eaa9bb4bca24df42214e1a05c18/1111777?chain=POLYGON&format=decimal", - MORALIS_API_ENDPOINT_TEST - ); - let response_meta = block_on(send_request_to_uri(uri_meta.as_str())).unwrap(); - let nft_moralis: NftFromMoralis = serde_json::from_str(&response_meta.to_string()).unwrap(); - assert_eq!(41237364, *nft_moralis.block_number_minted.unwrap()); - let token_uri = nft_moralis.common.token_uri.unwrap(); - let uri_response = block_on(send_request_to_uri(token_uri.as_str())).unwrap(); - serde_json::from_str::(&uri_response.to_string()).unwrap(); - } - - #[test] - fn test_antispam_api_requests() { - let uri_mnemonichq = format!( - "{}/api/blocklist/wallet/eth/0x3eb4b12127EdC81A4d2fD49658db07005bcAd065", - BLOCKLIST_API_ENDPOINT - ); - let mnemonichq_value = block_on(send_request_to_uri(uri_mnemonichq.as_str())).unwrap(); - let mnemonichq_res: MnemonicHQRes = serde_json::from_value(mnemonichq_value).unwrap(); - assert!(mnemonichq_res - .spam_contracts - .contains(&Address::from_str("0x0ded8542fc8b2b4e781b96e99fee6406550c9b7c").unwrap())); - assert_eq!(Chain::Eth, mnemonichq_res.network); - - let req_spam = SpamContractReq { - network: Chain::Eth, - addresses: "0x0ded8542fc8b2b4e781b96e99fee6406550c9b7c,0x8d1355b65da254f2cc4611453adfa8b7a13f60ee" - .to_string(), - }; - let req_json = serde_json::to_string(&req_spam).unwrap(); - let uri_contract = format!("{}/api/blocklist/contract/scan", BLOCKLIST_API_ENDPOINT); - let contract_scan_res = block_on(send_post_request_to_uri(uri_contract.as_str(), req_json)).unwrap(); - let spam_res: SpamContractRes = serde_json::from_slice(&contract_scan_res).unwrap(); - assert!(spam_res - .result - .get(&Address::from_str("0x0ded8542fc8b2b4e781b96e99fee6406550c9b7c").unwrap()) - .unwrap()); - assert!(spam_res - .result - .get(&Address::from_str("0x8d1355b65da254f2cc4611453adfa8b7a13f60ee").unwrap()) - .unwrap()); - - let req_phishing = PhishingDomainReq { - domains: "disposal-account-case-1f677.web.app,defi8090.vip".to_string(), - }; - let uri_domain = format!("{}/api/blocklist/domain/scan", BLOCKLIST_API_ENDPOINT); - let req_json = serde_json::to_string(&req_phishing).unwrap(); - let domain_scan_res = block_on(send_post_request_to_uri(uri_domain.as_str(), req_json)).unwrap(); - let phishing_res: PhishingDomainRes = serde_json::from_slice(&domain_scan_res).unwrap(); - assert!(phishing_res.result.get("disposal-account-case-1f677.web.app").unwrap()); - } - - #[test] - fn test_camo() { - let hex_token_uri = hex::encode("https://tikimetadata.s3.amazonaws.com/tiki_box.json"); - let uri_decode = format!("{}/url/decode/{}", BLOCKLIST_API_ENDPOINT, hex_token_uri); - let decode_res = block_on(send_request_to_uri(&uri_decode)).unwrap(); - let uri_meta: UriMeta = serde_json::from_value(decode_res).unwrap(); - assert_eq!( - uri_meta.raw_image_url.unwrap(), - "https://tikimetadata.s3.amazonaws.com/tiki_box.png" - ); - } - - #[test] - fn test_add_get_nfts() { block_on(test_add_get_nfts_impl()) } - - #[test] - fn test_last_nft_block() { block_on(test_last_nft_block_impl()) } - - #[test] - fn test_nft_list() { block_on(test_nft_list_impl()) } - - #[test] - fn test_remove_nft() { block_on(test_remove_nft_impl()) } - - #[test] - fn test_refresh_metadata() { block_on(test_refresh_metadata_impl()) } - - #[test] - fn test_nft_amount() { block_on(test_nft_amount_impl()) } - - #[test] - fn test_update_nft_spam_by_token_address() { block_on(test_update_nft_spam_by_token_address_impl()) } - - #[test] - fn test_exclude_nft_spam() { block_on(test_exclude_nft_spam_impl()) } - - #[test] - fn test_get_animation_external_domains() { block_on(test_get_animation_external_domains_impl()) } - - #[test] - fn test_update_nft_phishing_by_domain() { block_on(test_update_nft_phishing_by_domain_impl()) } - - #[test] - fn test_add_get_transfers() { block_on(test_add_get_transfers_impl()) } - - #[test] - fn test_last_transfer_block() { block_on(test_last_transfer_block_impl()) } - - #[test] - fn test_transfer_history() { block_on(test_transfer_history_impl()) } - - #[test] - fn test_transfer_history_filters() { block_on(test_transfer_history_filters_impl()) } - - #[test] - fn test_get_update_transfer_meta() { block_on(test_get_update_transfer_meta_impl()) } - - #[test] - fn test_update_transfer_spam_by_token_address() { block_on(test_update_transfer_spam_by_token_address_impl()) } - - #[test] - fn test_get_token_addresses() { block_on(test_get_token_addresses_impl()) } - - #[test] - fn test_exclude_transfer_spam() { block_on(test_exclude_transfer_spam_impl()) } - - #[test] - fn test_get_domains() { block_on(test_get_domains_impl()) } - - #[test] - fn test_update_transfer_phishing_by_domain() { block_on(test_update_transfer_phishing_by_domain_impl()) } -} +#[cfg(not(target_arch = "wasm32"))] +use mm2_net::native_http::send_request_to_uri; -#[cfg(target_arch = "wasm32")] -mod wasm_tests { - use crate::eth::eth_addr_to_hex; - use crate::nft::nft_structs::{Chain, MnemonicHQRes, NftFromMoralis, NftTransferHistoryFromMoralis, - PhishingDomainReq, PhishingDomainRes, SpamContractReq, SpamContractRes, UriMeta}; - use crate::nft::nft_tests::{BLOCKLIST_API_ENDPOINT, MORALIS_API_ENDPOINT_TEST, TEST_WALLET_ADDR_EVM}; - use crate::nft::storage::db_test_helpers::*; - use ethereum_types::Address; - use mm2_net::transport::send_post_request_to_uri; - use mm2_net::wasm_http::send_request_to_uri; - use std::str::FromStr; +common::cfg_wasm32! { use wasm_bindgen_test::*; + wasm_bindgen_test::wasm_bindgen_test_configure!(run_in_browser); + use mm2_net::wasm_http::send_request_to_uri; +} - wasm_bindgen_test_configure!(run_in_browser); - - #[wasm_bindgen_test] - async fn test_moralis_requests() { - let uri_nft_list = format!( - "{}/{}/nft?chain=POLYGON&format=decimal", - MORALIS_API_ENDPOINT_TEST, TEST_WALLET_ADDR_EVM - ); - let response_nft_list = send_request_to_uri(uri_nft_list.as_str()).await.unwrap(); - let nfts_list = response_nft_list["result"].as_array().unwrap(); - for nft_json in nfts_list { - let nft_moralis: NftFromMoralis = serde_json::from_str(&nft_json.to_string()).unwrap(); - assert_eq!(TEST_WALLET_ADDR_EVM, eth_addr_to_hex(&nft_moralis.common.owner_of)); - } - - let uri_history = format!( - "{}/{}/nft/transfers?chain=POLYGON&format=decimal", - MORALIS_API_ENDPOINT_TEST, TEST_WALLET_ADDR_EVM - ); - let response_transfer_history = send_request_to_uri(uri_history.as_str()).await.unwrap(); - let mut transfer_list = response_transfer_history["result"].as_array().unwrap().clone(); - assert!(!transfer_list.is_empty()); - let first_transfer = transfer_list.remove(transfer_list.len() - 1); - let transfer_moralis: NftTransferHistoryFromMoralis = - serde_json::from_str(&first_transfer.to_string()).unwrap(); - assert_eq!( - TEST_WALLET_ADDR_EVM, - eth_addr_to_hex(&transfer_moralis.common.to_address) - ); - - let uri_meta = format!( - "{}/nft/0xed55e4477b795eaa9bb4bca24df42214e1a05c18/1111777?chain=POLYGON&format=decimal", - MORALIS_API_ENDPOINT_TEST - ); - let response_meta = send_request_to_uri(uri_meta.as_str()).await.unwrap(); - let nft_moralis: NftFromMoralis = serde_json::from_str(&response_meta.to_string()).unwrap(); - assert_eq!(41237364, *nft_moralis.block_number_minted.unwrap()); - } - - #[wasm_bindgen_test] - async fn test_antispam_wallet_endpoint() { - let uri_mnemonichq = format!( - "{}/api/blocklist/wallet/eth/0x3eb4b12127EdC81A4d2fD49658db07005bcAd065", - BLOCKLIST_API_ENDPOINT - ); - let res_value = send_request_to_uri(uri_mnemonichq.as_str()).await.unwrap(); - let mnemonichq_res: MnemonicHQRes = serde_json::from_value(res_value).unwrap(); - assert!(mnemonichq_res - .spam_contracts - .contains(&Address::from_str("0x0ded8542fc8b2b4e781b96e99fee6406550c9b7c").unwrap())); - assert_eq!(Chain::Eth, mnemonichq_res.network); - } - - #[wasm_bindgen_test] - async fn test_antispam_scan_endpoints() { - let req_spam = SpamContractReq { - network: Chain::Eth, - addresses: "0x0ded8542fc8b2b4e781b96e99fee6406550c9b7c,0x8d1355b65da254f2cc4611453adfa8b7a13f60ee" - .to_string(), - }; - let uri_contract = format!("{}/api/blocklist/contract/scan", BLOCKLIST_API_ENDPOINT); - let req_json = serde_json::to_string(&req_spam).unwrap(); - let contract_scan_res = send_post_request_to_uri(uri_contract.as_str(), req_json).await.unwrap(); - let spam_res: SpamContractRes = serde_json::from_slice(&contract_scan_res).unwrap(); - assert!(spam_res - .result - .get(&Address::from_str("0x0ded8542fc8b2b4e781b96e99fee6406550c9b7c").unwrap()) - .unwrap()); - assert!(spam_res - .result - .get(&Address::from_str("0x8d1355b65da254f2cc4611453adfa8b7a13f60ee").unwrap()) - .unwrap()); - - let req_phishing = PhishingDomainReq { - domains: "disposal-account-case-1f677.web.app,defi8090.vip".to_string(), - }; - let req_json = serde_json::to_string(&req_phishing).unwrap(); - let uri_domain = format!("{}/api/blocklist/domain/scan", BLOCKLIST_API_ENDPOINT); - let domain_scan_res = send_post_request_to_uri(uri_domain.as_str(), req_json).await.unwrap(); - let phishing_res: PhishingDomainRes = serde_json::from_slice(&domain_scan_res).unwrap(); - assert!(phishing_res.result.get("disposal-account-case-1f677.web.app").unwrap()); - } - - #[wasm_bindgen_test] - async fn test_camo() { - let hex_token_uri = hex::encode("https://tikimetadata.s3.amazonaws.com/tiki_box.json"); - let uri_decode = format!("{}/url/decode/{}", BLOCKLIST_API_ENDPOINT, hex_token_uri); - let decode_res = send_request_to_uri(&uri_decode).await.unwrap(); - let uri_meta: UriMeta = serde_json::from_value(decode_res).unwrap(); - assert_eq!( - uri_meta.raw_image_url.unwrap(), - "https://tikimetadata.s3.amazonaws.com/tiki_box.png" - ); - } - - #[wasm_bindgen_test] - async fn test_add_get_nfts() { test_add_get_nfts_impl().await } - - #[wasm_bindgen_test] - async fn test_last_nft_block() { test_last_nft_block_impl().await } - - #[wasm_bindgen_test] - async fn test_nft_list() { test_nft_list_impl().await } - - #[wasm_bindgen_test] - async fn test_remove_nft() { test_remove_nft_impl().await } - - #[wasm_bindgen_test] - async fn test_nft_amount() { test_nft_amount_impl().await } - - #[wasm_bindgen_test] - async fn test_refresh_metadata() { test_refresh_metadata_impl().await } - - #[wasm_bindgen_test] - async fn test_update_nft_spam_by_token_address() { test_update_nft_spam_by_token_address_impl().await } - - #[wasm_bindgen_test] - async fn test_exclude_nft_spam() { test_exclude_nft_spam_impl().await } - - #[wasm_bindgen_test] - async fn test_get_animation_external_domains() { test_get_animation_external_domains_impl().await } - - #[wasm_bindgen_test] - async fn test_update_nft_phishing_by_domain() { test_update_nft_phishing_by_domain_impl().await } - - #[wasm_bindgen_test] - async fn test_add_get_transfers() { test_add_get_transfers_impl().await } - - #[wasm_bindgen_test] - async fn test_last_transfer_block() { test_last_transfer_block_impl().await } - - #[wasm_bindgen_test] - async fn test_transfer_history() { test_transfer_history_impl().await } - - #[wasm_bindgen_test] - async fn test_transfer_history_filters() { test_transfer_history_filters_impl().await } - - #[wasm_bindgen_test] - async fn test_get_update_transfer_meta() { test_get_update_transfer_meta_impl().await } - - #[wasm_bindgen_test] - async fn test_update_transfer_spam_by_token_address() { test_update_transfer_spam_by_token_address_impl().await } - - #[wasm_bindgen_test] - async fn test_get_token_addresses() { test_get_token_addresses_impl().await } - - #[wasm_bindgen_test] - async fn test_exclude_transfer_spam() { test_exclude_transfer_spam_impl().await } - - #[wasm_bindgen_test] - async fn test_get_domains() { test_get_domains_impl().await } +macro_rules! cross_test { + ($test_name:ident, $test_code:block) => { + #[cfg(not(target_arch = "wasm32"))] + #[tokio::test(flavor = "multi_thread")] + async fn $test_name() { $test_code } - #[wasm_bindgen_test] - async fn test_update_transfer_phishing_by_domain() { test_update_transfer_phishing_by_domain_impl().await } + #[cfg(target_arch = "wasm32")] + #[wasm_bindgen_test] + async fn $test_name() { $test_code } + }; } -#[cfg(any(test, target_arch = "wasm32"))] -mod nft_cross_tests { - use crate::eth::eth_addr_to_hex; - use crate::nft::nft_structs::{Chain, MnemonicHQRes, NftFromMoralis, NftListFilters, NftTransferHistoryFilters, - NftTransferHistoryFromMoralis, PhishingDomainReq, PhishingDomainRes, - SpamContractReq, SpamContractRes, TransferMeta, UriMeta}; - use crate::nft::storage::db_test_helpers::{init_nft_history_storage, init_nft_list_storage, nft, nft_list, - nft_transfer_history}; - use crate::nft::storage::{NftListStorageOps, NftTransferHistoryStorageOps, RemoveNftResult}; - use crate::nft::{check_moralis_ipfs_bafy, get_domain_from_url, process_metadata_for_spam_link, - process_text_for_spam_link}; - use ethereum_types::Address; - use mm2_net::transport::send_post_request_to_uri; - use mm2_number::BigDecimal; - use std::num::NonZeroUsize; - use std::str::FromStr; - - const MORALIS_API_ENDPOINT_TEST: &str = "https://moralis-proxy.komodo.earth/api/v2"; - const TEST_WALLET_ADDR_EVM: &str = "0x394d86994f954ed931b86791b62fe64f4c5dac37"; - const BLOCKLIST_API_ENDPOINT: &str = "https://nft.antispam.dragonhound.info"; - const TOKEN_ADD: &str = "0xfd913a305d70a60aac4faac70c739563738e1f81"; - const TOKEN_ID: &str = "214300044414"; - const TX_HASH: &str = "0x1e9f04e9b571b283bde02c98c2a97da39b2bb665b57c1f2b0b733f9b681debbe"; - const LOG_INDEX: u32 = 495; - - #[cfg(not(target_arch = "wasm32"))] - use mm2_net::native_http::send_request_to_uri; - - common::cfg_wasm32! { - use wasm_bindgen_test::*; - wasm_bindgen_test::wasm_bindgen_test_configure!(run_in_browser); - use mm2_net::wasm_http::send_request_to_uri; +cross_test!(test_moralis_ipfs_bafy, { + let uri = "https://ipfs.moralis.io:2053/ipfs/bafybeifnek24coy5xj5qabdwh24dlp5omq34nzgvazkfyxgnqms4eidsiq/1.json"; + let res_uri = check_moralis_ipfs_bafy(Some(uri)); + let expected = "https://ipfs.io/ipfs/bafybeifnek24coy5xj5qabdwh24dlp5omq34nzgvazkfyxgnqms4eidsiq/1.json"; + assert_eq!(expected, res_uri.unwrap()); +}); + +cross_test!(test_get_domain_from_url, { + let image_url = "https://public.nftstatic.com/static/nft/res/4df0a5da04174e1e9be04b22a805f605.png"; + let res_domain = get_domain_from_url(Some(image_url)); + let expected = "public.nftstatic.com"; + assert_eq!(expected, res_domain.unwrap()); +}); + +cross_test!(test_invalid_moralis_ipfs_link, { + let uri = "example.com/bafy?1=ipfs.moralis.io&e=https://"; + let res_uri = check_moralis_ipfs_bafy(Some(uri)); + assert_eq!(uri, res_uri.unwrap()); +}); + +cross_test!(test_check_for_spam_links, { + let mut spam_text = Some("https://arweave.net".to_string()); + assert!(process_text_for_spam_link(&mut spam_text, true).unwrap()); + let url_redacted = "URL redacted for user protection"; + assert_eq!(url_redacted, spam_text.unwrap()); + + let mut spam_text = Some("ftp://123path ".to_string()); + assert!(process_text_for_spam_link(&mut spam_text, true).unwrap()); + let url_redacted = "URL redacted for user protection"; + assert_eq!(url_redacted, spam_text.unwrap()); + + let mut spam_text = Some("/192.168.1.1/some.example.org?type=A".to_string()); + assert!(process_text_for_spam_link(&mut spam_text, true).unwrap()); + let url_redacted = "URL redacted for user protection"; + assert_eq!(url_redacted, spam_text.unwrap()); + + let mut spam_text = Some(r"C:\Users\path\".to_string()); + assert!(process_text_for_spam_link(&mut spam_text, true).unwrap()); + let url_redacted = "URL redacted for user protection"; + assert_eq!(url_redacted, spam_text.unwrap()); + + let mut valid_text = Some("Hello my name is NFT (The best ever!)".to_string()); + assert!(!process_text_for_spam_link(&mut valid_text, true).unwrap()); + assert_eq!("Hello my name is NFT (The best ever!)", valid_text.unwrap()); + + let mut nft = nft(); + assert!(process_metadata_for_spam_link(&mut nft, true).unwrap()); + let meta_redacted = "{\"name\":\"URL redacted for user protection\",\"image\":\"https://tikimetadata.s3.amazonaws.com/tiki_box.png\"}"; + assert_eq!(meta_redacted, nft.common.metadata.unwrap()) +}); + +cross_test!(test_moralis_requests, { + let uri_nft_list = format!( + "{}/{}/nft?chain=POLYGON&format=decimal", + MORALIS_API_ENDPOINT_TEST, TEST_WALLET_ADDR_EVM + ); + let response_nft_list = send_request_to_uri(uri_nft_list.as_str()).await.unwrap(); + let nfts_list = response_nft_list["result"].as_array().unwrap(); + for nft_json in nfts_list { + let nft_moralis: NftFromMoralis = serde_json::from_str(&nft_json.to_string()).unwrap(); + assert_eq!(TEST_WALLET_ADDR_EVM, eth_addr_to_hex(&nft_moralis.common.owner_of)); } - macro_rules! cross_test { - ($test_name:ident, $test_code:block) => { - #[cfg(not(target_arch = "wasm32"))] - #[tokio::test(flavor = "multi_thread")] - async fn $test_name() { $test_code } - - #[cfg(target_arch = "wasm32")] - #[wasm_bindgen_test] - async fn $test_name() { $test_code } - }; + let uri_history = format!( + "{}/{}/nft/transfers?chain=POLYGON&format=decimal", + MORALIS_API_ENDPOINT_TEST, TEST_WALLET_ADDR_EVM + ); + let response_transfer_history = send_request_to_uri(uri_history.as_str()).await.unwrap(); + let mut transfer_list = response_transfer_history["result"].as_array().unwrap().clone(); + assert!(!transfer_list.is_empty()); + let first_transfer = transfer_list.remove(transfer_list.len() - 1); + let transfer_moralis: NftTransferHistoryFromMoralis = serde_json::from_str(&first_transfer.to_string()).unwrap(); + assert_eq!( + TEST_WALLET_ADDR_EVM, + eth_addr_to_hex(&transfer_moralis.common.to_address) + ); + + let uri_meta = format!( + "{}/nft/0xed55e4477b795eaa9bb4bca24df42214e1a05c18/1111777?chain=POLYGON&format=decimal", + MORALIS_API_ENDPOINT_TEST + ); + let response_meta = send_request_to_uri(uri_meta.as_str()).await.unwrap(); + let nft_moralis: NftFromMoralis = serde_json::from_str(&response_meta.to_string()).unwrap(); + assert_eq!(41237364, *nft_moralis.block_number_minted.unwrap()); +}); + +cross_test!(test_antispam_wallet_endpoint, { + let uri_mnemonichq = format!( + "{}/api/blocklist/wallet/eth/0x3eb4b12127EdC81A4d2fD49658db07005bcAd065", + BLOCKLIST_API_ENDPOINT + ); + let res_value = send_request_to_uri(uri_mnemonichq.as_str()).await.unwrap(); + let mnemonichq_res: MnemonicHQRes = serde_json::from_value(res_value).unwrap(); + assert!(mnemonichq_res + .spam_contracts + .contains(&Address::from_str("0x0ded8542fc8b2b4e781b96e99fee6406550c9b7c").unwrap())); + assert_eq!(Chain::Eth, mnemonichq_res.network); +}); + +cross_test!(test_antispam_scan_endpoints, { + let req_spam = SpamContractReq { + network: Chain::Eth, + addresses: "0x0ded8542fc8b2b4e781b96e99fee6406550c9b7c,0x8d1355b65da254f2cc4611453adfa8b7a13f60ee".to_string(), + }; + let uri_contract = format!("{}/api/blocklist/contract/scan", BLOCKLIST_API_ENDPOINT); + let req_json = serde_json::to_string(&req_spam).unwrap(); + let contract_scan_res = send_post_request_to_uri(uri_contract.as_str(), req_json).await.unwrap(); + let spam_res: SpamContractRes = serde_json::from_slice(&contract_scan_res).unwrap(); + assert!(spam_res + .result + .get(&Address::from_str("0x0ded8542fc8b2b4e781b96e99fee6406550c9b7c").unwrap()) + .unwrap()); + assert!(spam_res + .result + .get(&Address::from_str("0x8d1355b65da254f2cc4611453adfa8b7a13f60ee").unwrap()) + .unwrap()); + + let req_phishing = PhishingDomainReq { + domains: "disposal-account-case-1f677.web.app,defi8090.vip".to_string(), + }; + let req_json = serde_json::to_string(&req_phishing).unwrap(); + let uri_domain = format!("{}/api/blocklist/domain/scan", BLOCKLIST_API_ENDPOINT); + let domain_scan_res = send_post_request_to_uri(uri_domain.as_str(), req_json).await.unwrap(); + let phishing_res: PhishingDomainRes = serde_json::from_slice(&domain_scan_res).unwrap(); + assert!(phishing_res.result.get("disposal-account-case-1f677.web.app").unwrap()); +}); + +cross_test!(test_camo, { + let hex_token_uri = hex::encode("https://tikimetadata.s3.amazonaws.com/tiki_box.json"); + let uri_decode = format!("{}/url/decode/{}", BLOCKLIST_API_ENDPOINT, hex_token_uri); + let decode_res = send_request_to_uri(&uri_decode).await.unwrap(); + let uri_meta: UriMeta = serde_json::from_value(decode_res).unwrap(); + assert_eq!( + uri_meta.raw_image_url.unwrap(), + "https://tikimetadata.s3.amazonaws.com/tiki_box.png" + ); +}); + +cross_test!(test_add_get_nfts, { + let chain = Chain::Bsc; + let storage = init_nft_list_storage(&chain).await; + let nft_list = nft_list(); + storage.add_nfts_to_list(chain, nft_list, 28056726).await.unwrap(); + + let token_id = BigDecimal::from_str(TOKEN_ID).unwrap(); + let nft = storage + .get_nft(&chain, TOKEN_ADD.to_string(), token_id) + .await + .unwrap() + .unwrap(); + assert_eq!(nft.block_number, 28056721); +}); + +cross_test!(test_last_nft_block, { + let chain = Chain::Bsc; + let storage = init_nft_list_storage(&chain).await; + let nft_list = nft_list(); + storage.add_nfts_to_list(chain, nft_list, 28056726).await.unwrap(); + + let last_block = NftListStorageOps::get_last_block_number(&storage, &chain) + .await + .unwrap() + .unwrap(); + assert_eq!(last_block, 28056726); +}); + +cross_test!(test_nft_list, { + let chain = Chain::Bsc; + let storage = init_nft_list_storage(&chain).await; + let nft_list = nft_list(); + storage.add_nfts_to_list(chain, nft_list, 28056726).await.unwrap(); + + let nft_list = storage + .get_nft_list(vec![chain], false, 1, Some(NonZeroUsize::new(3).unwrap()), None) + .await + .unwrap(); + assert_eq!(nft_list.nfts.len(), 1); + let nft = nft_list.nfts.get(0).unwrap(); + assert_eq!(nft.block_number, 28056721); + assert_eq!(nft_list.skipped, 2); + assert_eq!(nft_list.total, 4); +}); + +cross_test!(test_remove_nft, { + let chain = Chain::Bsc; + let storage = init_nft_list_storage(&chain).await; + let nft_list = nft_list(); + storage.add_nfts_to_list(chain, nft_list, 28056726).await.unwrap(); + + let token_id = BigDecimal::from_str(TOKEN_ID).unwrap(); + let remove_rslt = storage + .remove_nft_from_list(&chain, TOKEN_ADD.to_string(), token_id, 28056800) + .await + .unwrap(); + assert_eq!(remove_rslt, RemoveNftResult::NftRemoved); + let list_len = storage + .get_nft_list(vec![chain], true, 1, None, None) + .await + .unwrap() + .nfts + .len(); + assert_eq!(list_len, 3); + let last_scanned_block = storage.get_last_scanned_block(&chain).await.unwrap().unwrap(); + assert_eq!(last_scanned_block, 28056800); +}); + +cross_test!(test_nft_amount, { + let chain = Chain::Bsc; + let storage = init_nft_list_storage(&chain).await; + let mut nft = nft(); + storage + .add_nfts_to_list(chain, vec![nft.clone()], 25919780) + .await + .unwrap(); + + nft.common.amount -= BigDecimal::from(1); + storage.update_nft_amount(&chain, nft.clone(), 25919800).await.unwrap(); + let amount = storage + .get_nft_amount( + &chain, + eth_addr_to_hex(&nft.common.token_address), + nft.common.token_id.clone(), + ) + .await + .unwrap() + .unwrap(); + assert_eq!(amount, "1"); + let last_scanned_block = storage.get_last_scanned_block(&chain).await.unwrap().unwrap(); + assert_eq!(last_scanned_block, 25919800); + + nft.common.amount += BigDecimal::from(1); + nft.block_number = 25919900; + storage + .update_nft_amount_and_block_number(&chain, nft.clone()) + .await + .unwrap(); + let amount = storage + .get_nft_amount(&chain, eth_addr_to_hex(&nft.common.token_address), nft.common.token_id) + .await + .unwrap() + .unwrap(); + assert_eq!(amount, "2"); + let last_scanned_block = storage.get_last_scanned_block(&chain).await.unwrap().unwrap(); + assert_eq!(last_scanned_block, 25919900); +}); + +cross_test!(test_refresh_metadata, { + let chain = Chain::Bsc; + let storage = init_nft_list_storage(&chain).await; + let new_symbol = "NEW_SYMBOL"; + let mut nft = nft(); + storage + .add_nfts_to_list(chain, vec![nft.clone()], 25919780) + .await + .unwrap(); + nft.common.symbol = Some(new_symbol.to_string()); + drop_mutability!(nft); + let token_add = eth_addr_to_hex(&nft.common.token_address); + let token_id = nft.common.token_id.clone(); + storage.refresh_nft_metadata(&chain, nft).await.unwrap(); + let nft_upd = storage.get_nft(&chain, token_add, token_id).await.unwrap().unwrap(); + assert_eq!(new_symbol.to_string(), nft_upd.common.symbol.unwrap()); +}); + +cross_test!(test_update_nft_spam_by_token_address, { + let chain = Chain::Bsc; + let storage = init_nft_list_storage(&chain).await; + let nft_list = nft_list(); + storage.add_nfts_to_list(chain, nft_list, 28056726).await.unwrap(); + + storage + .update_nft_spam_by_token_address(&chain, TOKEN_ADD.to_string(), true) + .await + .unwrap(); + let nfts = storage + .get_nfts_by_token_address(chain, TOKEN_ADD.to_string()) + .await + .unwrap(); + for nft in nfts { + assert!(nft.common.possible_spam); } - - cross_test!(test_moralis_ipfs_bafy, { - let uri = - "https://ipfs.moralis.io:2053/ipfs/bafybeifnek24coy5xj5qabdwh24dlp5omq34nzgvazkfyxgnqms4eidsiq/1.json"; - let res_uri = check_moralis_ipfs_bafy(Some(uri)); - let expected = "https://ipfs.io/ipfs/bafybeifnek24coy5xj5qabdwh24dlp5omq34nzgvazkfyxgnqms4eidsiq/1.json"; - assert_eq!(expected, res_uri.unwrap()); - }); - - cross_test!(test_get_domain_from_url, { - let image_url = "https://public.nftstatic.com/static/nft/res/4df0a5da04174e1e9be04b22a805f605.png"; - let res_domain = get_domain_from_url(Some(image_url)); - let expected = "public.nftstatic.com"; - assert_eq!(expected, res_domain.unwrap()); - }); - - cross_test!(test_invalid_moralis_ipfs_link, { - let uri = "example.com/bafy?1=ipfs.moralis.io&e=https://"; - let res_uri = check_moralis_ipfs_bafy(Some(uri)); - assert_eq!(uri, res_uri.unwrap()); - }); - - cross_test!(test_check_for_spam_links, { - let mut spam_text = Some("https://arweave.net".to_string()); - assert!(process_text_for_spam_link(&mut spam_text, true).unwrap()); - let url_redacted = "URL redacted for user protection"; - assert_eq!(url_redacted, spam_text.unwrap()); - - let mut spam_text = Some("ftp://123path ".to_string()); - assert!(process_text_for_spam_link(&mut spam_text, true).unwrap()); - let url_redacted = "URL redacted for user protection"; - assert_eq!(url_redacted, spam_text.unwrap()); - - let mut spam_text = Some("/192.168.1.1/some.example.org?type=A".to_string()); - assert!(process_text_for_spam_link(&mut spam_text, true).unwrap()); - let url_redacted = "URL redacted for user protection"; - assert_eq!(url_redacted, spam_text.unwrap()); - - let mut spam_text = Some(r"C:\Users\path\".to_string()); - assert!(process_text_for_spam_link(&mut spam_text, true).unwrap()); - let url_redacted = "URL redacted for user protection"; - assert_eq!(url_redacted, spam_text.unwrap()); - - let mut valid_text = Some("Hello my name is NFT (The best ever!)".to_string()); - assert!(!process_text_for_spam_link(&mut valid_text, true).unwrap()); - assert_eq!("Hello my name is NFT (The best ever!)", valid_text.unwrap()); - - let mut nft = nft(); - assert!(process_metadata_for_spam_link(&mut nft, true).unwrap()); - let meta_redacted = "{\"name\":\"URL redacted for user protection\",\"image\":\"https://tikimetadata.s3.amazonaws.com/tiki_box.png\"}"; - assert_eq!(meta_redacted, nft.common.metadata.unwrap()) - }); - - cross_test!(test_moralis_requests, { - let uri_nft_list = format!( - "{}/{}/nft?chain=POLYGON&format=decimal", - MORALIS_API_ENDPOINT_TEST, TEST_WALLET_ADDR_EVM - ); - let response_nft_list = send_request_to_uri(uri_nft_list.as_str()).await.unwrap(); - let nfts_list = response_nft_list["result"].as_array().unwrap(); - for nft_json in nfts_list { - let nft_moralis: NftFromMoralis = serde_json::from_str(&nft_json.to_string()).unwrap(); - assert_eq!(TEST_WALLET_ADDR_EVM, eth_addr_to_hex(&nft_moralis.common.owner_of)); - } - - let uri_history = format!( - "{}/{}/nft/transfers?chain=POLYGON&format=decimal", - MORALIS_API_ENDPOINT_TEST, TEST_WALLET_ADDR_EVM - ); - let response_transfer_history = send_request_to_uri(uri_history.as_str()).await.unwrap(); - let mut transfer_list = response_transfer_history["result"].as_array().unwrap().clone(); - assert!(!transfer_list.is_empty()); - let first_transfer = transfer_list.remove(transfer_list.len() - 1); - let transfer_moralis: NftTransferHistoryFromMoralis = - serde_json::from_str(&first_transfer.to_string()).unwrap(); - assert_eq!( - TEST_WALLET_ADDR_EVM, - eth_addr_to_hex(&transfer_moralis.common.to_address) - ); - - let uri_meta = format!( - "{}/nft/0xed55e4477b795eaa9bb4bca24df42214e1a05c18/1111777?chain=POLYGON&format=decimal", - MORALIS_API_ENDPOINT_TEST - ); - let response_meta = send_request_to_uri(uri_meta.as_str()).await.unwrap(); - let nft_moralis: NftFromMoralis = serde_json::from_str(&response_meta.to_string()).unwrap(); - assert_eq!(41237364, *nft_moralis.block_number_minted.unwrap()); - }); - - cross_test!(test_antispam_wallet_endpoint, { - let uri_mnemonichq = format!( - "{}/api/blocklist/wallet/eth/0x3eb4b12127EdC81A4d2fD49658db07005bcAd065", - BLOCKLIST_API_ENDPOINT - ); - let res_value = send_request_to_uri(uri_mnemonichq.as_str()).await.unwrap(); - let mnemonichq_res: MnemonicHQRes = serde_json::from_value(res_value).unwrap(); - assert!(mnemonichq_res - .spam_contracts - .contains(&Address::from_str("0x0ded8542fc8b2b4e781b96e99fee6406550c9b7c").unwrap())); - assert_eq!(Chain::Eth, mnemonichq_res.network); - }); - - cross_test!(test_antispam_scan_endpoints, { - let req_spam = SpamContractReq { - network: Chain::Eth, - addresses: "0x0ded8542fc8b2b4e781b96e99fee6406550c9b7c,0x8d1355b65da254f2cc4611453adfa8b7a13f60ee" - .to_string(), - }; - let uri_contract = format!("{}/api/blocklist/contract/scan", BLOCKLIST_API_ENDPOINT); - let req_json = serde_json::to_string(&req_spam).unwrap(); - let contract_scan_res = send_post_request_to_uri(uri_contract.as_str(), req_json).await.unwrap(); - let spam_res: SpamContractRes = serde_json::from_slice(&contract_scan_res).unwrap(); - assert!(spam_res - .result - .get(&Address::from_str("0x0ded8542fc8b2b4e781b96e99fee6406550c9b7c").unwrap()) - .unwrap()); - assert!(spam_res - .result - .get(&Address::from_str("0x8d1355b65da254f2cc4611453adfa8b7a13f60ee").unwrap()) - .unwrap()); - - let req_phishing = PhishingDomainReq { - domains: "disposal-account-case-1f677.web.app,defi8090.vip".to_string(), - }; - let req_json = serde_json::to_string(&req_phishing).unwrap(); - let uri_domain = format!("{}/api/blocklist/domain/scan", BLOCKLIST_API_ENDPOINT); - let domain_scan_res = send_post_request_to_uri(uri_domain.as_str(), req_json).await.unwrap(); - let phishing_res: PhishingDomainRes = serde_json::from_slice(&domain_scan_res).unwrap(); - assert!(phishing_res.result.get("disposal-account-case-1f677.web.app").unwrap()); - }); - - cross_test!(test_camo, { - let hex_token_uri = hex::encode("https://tikimetadata.s3.amazonaws.com/tiki_box.json"); - let uri_decode = format!("{}/url/decode/{}", BLOCKLIST_API_ENDPOINT, hex_token_uri); - let decode_res = send_request_to_uri(&uri_decode).await.unwrap(); - let uri_meta: UriMeta = serde_json::from_value(decode_res).unwrap(); - assert_eq!( - uri_meta.raw_image_url.unwrap(), - "https://tikimetadata.s3.amazonaws.com/tiki_box.png" - ); - }); - - ///// - - cross_test!(test_add_get_nfts, { - let chain = Chain::Bsc; - let storage = init_nft_list_storage(&chain).await; - let nft_list = nft_list(); - storage.add_nfts_to_list(chain, nft_list, 28056726).await.unwrap(); - - let token_id = BigDecimal::from_str(TOKEN_ID).unwrap(); - let nft = storage - .get_nft(&chain, TOKEN_ADD.to_string(), token_id) - .await - .unwrap() - .unwrap(); - assert_eq!(nft.block_number, 28056721); - }); - - cross_test!(test_last_nft_block, { - let chain = Chain::Bsc; - let storage = init_nft_list_storage(&chain).await; - let nft_list = nft_list(); - storage.add_nfts_to_list(chain, nft_list, 28056726).await.unwrap(); - - let last_block = NftListStorageOps::get_last_block_number(&storage, &chain) - .await - .unwrap() - .unwrap(); - assert_eq!(last_block, 28056726); - }); - - cross_test!(test_nft_list, { - let chain = Chain::Bsc; - let storage = init_nft_list_storage(&chain).await; - let nft_list = nft_list(); - storage.add_nfts_to_list(chain, nft_list, 28056726).await.unwrap(); - - let nft_list = storage - .get_nft_list(vec![chain], false, 1, Some(NonZeroUsize::new(3).unwrap()), None) - .await - .unwrap(); - assert_eq!(nft_list.nfts.len(), 1); - let nft = nft_list.nfts.get(0).unwrap(); - assert_eq!(nft.block_number, 28056721); - assert_eq!(nft_list.skipped, 2); - assert_eq!(nft_list.total, 4); - }); - - cross_test!(test_remove_nft, { - let chain = Chain::Bsc; - let storage = init_nft_list_storage(&chain).await; - let nft_list = nft_list(); - storage.add_nfts_to_list(chain, nft_list, 28056726).await.unwrap(); - - let token_id = BigDecimal::from_str(TOKEN_ID).unwrap(); - let remove_rslt = storage - .remove_nft_from_list(&chain, TOKEN_ADD.to_string(), token_id, 28056800) - .await - .unwrap(); - assert_eq!(remove_rslt, RemoveNftResult::NftRemoved); - let list_len = storage - .get_nft_list(vec![chain], true, 1, None, None) - .await - .unwrap() - .nfts - .len(); - assert_eq!(list_len, 3); - let last_scanned_block = storage.get_last_scanned_block(&chain).await.unwrap().unwrap(); - assert_eq!(last_scanned_block, 28056800); - }); - - cross_test!(test_nft_amount, { - let chain = Chain::Bsc; - let storage = init_nft_list_storage(&chain).await; - let mut nft = nft(); - storage - .add_nfts_to_list(chain, vec![nft.clone()], 25919780) - .await - .unwrap(); - - nft.common.amount -= BigDecimal::from(1); - storage.update_nft_amount(&chain, nft.clone(), 25919800).await.unwrap(); - let amount = storage - .get_nft_amount( - &chain, - eth_addr_to_hex(&nft.common.token_address), - nft.common.token_id.clone(), - ) - .await - .unwrap() - .unwrap(); - assert_eq!(amount, "1"); - let last_scanned_block = storage.get_last_scanned_block(&chain).await.unwrap().unwrap(); - assert_eq!(last_scanned_block, 25919800); - - nft.common.amount += BigDecimal::from(1); - nft.block_number = 25919900; +}); + +cross_test!(test_exclude_nft_spam, { + let chain = Chain::Bsc; + let storage = init_nft_list_storage(&chain).await; + let nft_list = nft_list(); + storage.add_nfts_to_list(chain, nft_list, 28056726).await.unwrap(); + + let filters = NftListFilters { + exclude_spam: true, + exclude_phishing: false, + }; + let nft_list = storage + .get_nft_list(vec![chain], true, 1, None, Some(filters)) + .await + .unwrap(); + assert_eq!(nft_list.nfts.len(), 3); +}); + +cross_test!(test_get_animation_external_domains, { + let chain = Chain::Bsc; + let storage = init_nft_list_storage(&chain).await; + let nft_list = nft_list(); + storage.add_nfts_to_list(chain, nft_list, 28056726).await.unwrap(); + + let domains = storage.get_animation_external_domains(&chain).await.unwrap(); + assert_eq!(2, domains.len()); + assert!(domains.contains("tikimetadata.s3.amazonaws.com")); + assert!(domains.contains("public.nftstatic.com")); +}); + +cross_test!(test_update_nft_phishing_by_domain, { + let chain = Chain::Bsc; + let storage = init_nft_list_storage(&chain).await; + let nft_list = nft_list(); + storage.add_nfts_to_list(chain, nft_list, 28056726).await.unwrap(); + + let domains = vec![ + "tikimetadata.s3.amazonaws.com".to_string(), + "public.nftstatic.com".to_string(), + ]; + for domain in domains.into_iter() { storage - .update_nft_amount_and_block_number(&chain, nft.clone()) - .await - .unwrap(); - let amount = storage - .get_nft_amount(&chain, eth_addr_to_hex(&nft.common.token_address), nft.common.token_id) + .update_nft_phishing_by_domain(&chain, domain, true) .await - .unwrap() .unwrap(); - assert_eq!(amount, "2"); - let last_scanned_block = storage.get_last_scanned_block(&chain).await.unwrap().unwrap(); - assert_eq!(last_scanned_block, 25919900); - }); - - cross_test!(test_refresh_metadata, { - let chain = Chain::Bsc; - let storage = init_nft_list_storage(&chain).await; - let new_symbol = "NEW_SYMBOL"; - let mut nft = nft(); - storage - .add_nfts_to_list(chain, vec![nft.clone()], 25919780) - .await - .unwrap(); - nft.common.symbol = Some(new_symbol.to_string()); - drop_mutability!(nft); - let token_add = eth_addr_to_hex(&nft.common.token_address); - let token_id = nft.common.token_id.clone(); - storage.refresh_nft_metadata(&chain, nft).await.unwrap(); - let nft_upd = storage.get_nft(&chain, token_add, token_id).await.unwrap().unwrap(); - assert_eq!(new_symbol.to_string(), nft_upd.common.symbol.unwrap()); - }); - - cross_test!(test_update_nft_spam_by_token_address, { - let chain = Chain::Bsc; - let storage = init_nft_list_storage(&chain).await; - let nft_list = nft_list(); - storage.add_nfts_to_list(chain, nft_list, 28056726).await.unwrap(); - - storage - .update_nft_spam_by_token_address(&chain, TOKEN_ADD.to_string(), true) - .await - .unwrap(); - let nfts = storage - .get_nfts_by_token_address(chain, TOKEN_ADD.to_string()) - .await - .unwrap(); - for nft in nfts { - assert!(nft.common.possible_spam); - } - }); - - cross_test!(test_exclude_nft_spam, { - let chain = Chain::Bsc; - let storage = init_nft_list_storage(&chain).await; - let nft_list = nft_list(); - storage.add_nfts_to_list(chain, nft_list, 28056726).await.unwrap(); - - let filters = NftListFilters { - exclude_spam: true, - exclude_phishing: false, - }; - let nft_list = storage - .get_nft_list(vec![chain], true, 1, None, Some(filters)) - .await - .unwrap(); - assert_eq!(nft_list.nfts.len(), 3); - }); - - cross_test!(test_get_animation_external_domains, { - let chain = Chain::Bsc; - let storage = init_nft_list_storage(&chain).await; - let nft_list = nft_list(); - storage.add_nfts_to_list(chain, nft_list, 28056726).await.unwrap(); - - let domains = storage.get_animation_external_domains(&chain).await.unwrap(); - assert_eq!(2, domains.len()); - assert!(domains.contains("tikimetadata.s3.amazonaws.com")); - assert!(domains.contains("public.nftstatic.com")); - }); - - cross_test!(test_update_nft_phishing_by_domain, { - let chain = Chain::Bsc; - let storage = init_nft_list_storage(&chain).await; - let nft_list = nft_list(); - storage.add_nfts_to_list(chain, nft_list, 28056726).await.unwrap(); - - let domains = vec![ - "tikimetadata.s3.amazonaws.com".to_string(), - "public.nftstatic.com".to_string(), - ]; - for domain in domains.into_iter() { - storage - .update_nft_phishing_by_domain(&chain, domain, true) - .await - .unwrap(); - } - let nfts = storage - .get_nft_list(vec![chain], true, 1, None, None) - .await - .unwrap() - .nfts; - for nft in nfts.into_iter() { - assert!(nft.possible_phishing); - } - }); - - cross_test!(test_add_get_transfers, { - let chain = Chain::Bsc; - let storage = init_nft_history_storage(&chain).await; - let transfers = nft_transfer_history(); - storage.add_transfers_to_history(chain, transfers).await.unwrap(); - - let token_id = BigDecimal::from_str(TOKEN_ID).unwrap(); - let transfer1 = storage - .get_transfers_by_token_addr_id(chain, TOKEN_ADD.to_string(), token_id) - .await - .unwrap() - .get(0) - .unwrap() - .clone(); - assert_eq!(transfer1.block_number, 28056721); - let transfer2 = storage - .get_transfer_by_tx_hash_and_log_index(&chain, TX_HASH.to_string(), LOG_INDEX) - .await - .unwrap() - .unwrap(); - assert_eq!(transfer2.block_number, 28056726); - let transfer_from = storage.get_transfers_from_block(chain, 28056721).await.unwrap(); - assert_eq!(transfer_from.len(), 3); - }); - - cross_test!(test_last_transfer_block, { - let chain = Chain::Bsc; - let storage = init_nft_history_storage(&chain).await; - let transfers = nft_transfer_history(); - storage.add_transfers_to_history(chain, transfers).await.unwrap(); - - let last_block = NftTransferHistoryStorageOps::get_last_block_number(&storage, &chain) - .await - .unwrap() - .unwrap(); - assert_eq!(last_block, 28056726); - }); - - cross_test!(test_transfer_history, { - let chain = Chain::Bsc; - let storage = init_nft_history_storage(&chain).await; - let transfers = nft_transfer_history(); - storage.add_transfers_to_history(chain, transfers).await.unwrap(); - - let transfer_history = storage - .get_transfer_history(vec![chain], false, 1, Some(NonZeroUsize::new(3).unwrap()), None) - .await - .unwrap(); - assert_eq!(transfer_history.transfer_history.len(), 1); - let transfer = transfer_history.transfer_history.get(0).unwrap(); - assert_eq!(transfer.block_number, 28056721); - assert_eq!(transfer_history.skipped, 2); - assert_eq!(transfer_history.total, 4); - }); - - cross_test!(test_transfer_history_filters, { - let chain = Chain::Bsc; - let storage = init_nft_history_storage(&chain).await; - let transfers = nft_transfer_history(); - storage.add_transfers_to_history(chain, transfers).await.unwrap(); - - let filters = NftTransferHistoryFilters { - receive: true, - send: false, - from_date: None, - to_date: None, - exclude_spam: false, - exclude_phishing: false, - }; - - let filters1 = NftTransferHistoryFilters { - receive: false, - send: false, - from_date: None, - to_date: Some(1677166110), - exclude_spam: false, - exclude_phishing: false, - }; - - let filters2 = NftTransferHistoryFilters { - receive: false, - send: false, - from_date: Some(1677166110), - to_date: Some(1683627417), - exclude_spam: false, - exclude_phishing: false, - }; - - let transfer_history = storage - .get_transfer_history(vec![chain], true, 1, None, Some(filters)) - .await - .unwrap(); - assert_eq!(transfer_history.transfer_history.len(), 4); - let transfer = transfer_history.transfer_history.get(0).unwrap(); - assert_eq!(transfer.block_number, 28056726); - - let transfer_history1 = storage - .get_transfer_history(vec![chain], true, 1, None, Some(filters1)) - .await - .unwrap(); - assert_eq!(transfer_history1.transfer_history.len(), 1); - let transfer1 = transfer_history1.transfer_history.get(0).unwrap(); - assert_eq!(transfer1.block_number, 25919780); - - let transfer_history2 = storage - .get_transfer_history(vec![chain], true, 1, None, Some(filters2)) - .await - .unwrap(); - assert_eq!(transfer_history2.transfer_history.len(), 2); - let transfer_0 = transfer_history2.transfer_history.get(0).unwrap(); - assert_eq!(transfer_0.block_number, 28056721); - let transfer_1 = transfer_history2.transfer_history.get(1).unwrap(); - assert_eq!(transfer_1.block_number, 25919780); - }); - - cross_test!(test_get_update_transfer_meta, { - let chain = Chain::Bsc; - let storage = init_nft_history_storage(&chain).await; - let transfers = nft_transfer_history(); - storage.add_transfers_to_history(chain, transfers).await.unwrap(); - - let vec_token_add_id = storage.get_transfers_with_empty_meta(chain).await.unwrap(); - assert_eq!(vec_token_add_id.len(), 3); - - let token_add = "0x5c7d6712dfaf0cb079d48981781c8705e8417ca0".to_string(); - let transfer_meta = TransferMeta { - token_address: token_add.clone(), - token_id: Default::default(), - token_uri: None, - token_domain: None, - collection_name: None, - image_url: None, - image_domain: None, - token_name: Some("Tiki box".to_string()), - }; - storage - .update_transfers_meta_by_token_addr_id(&chain, transfer_meta, true) - .await - .unwrap(); - let transfer_upd = storage - .get_transfers_by_token_addr_id(chain, token_add, Default::default()) - .await - .unwrap(); - let transfer_upd = transfer_upd.get(0).unwrap(); - assert_eq!(transfer_upd.token_name, Some("Tiki box".to_string())); - assert!(transfer_upd.common.possible_spam); - }); - - cross_test!(test_update_transfer_spam_by_token_address, { - let chain = Chain::Bsc; - let storage = init_nft_history_storage(&chain).await; - let transfers = nft_transfer_history(); - storage.add_transfers_to_history(chain, transfers).await.unwrap(); - + } + let nfts = storage + .get_nft_list(vec![chain], true, 1, None, None) + .await + .unwrap() + .nfts; + for nft in nfts.into_iter() { + assert!(nft.possible_phishing); + } +}); + +cross_test!(test_add_get_transfers, { + let chain = Chain::Bsc; + let storage = init_nft_history_storage(&chain).await; + let transfers = nft_transfer_history(); + storage.add_transfers_to_history(chain, transfers).await.unwrap(); + + let token_id = BigDecimal::from_str(TOKEN_ID).unwrap(); + let transfer1 = storage + .get_transfers_by_token_addr_id(chain, TOKEN_ADD.to_string(), token_id) + .await + .unwrap() + .get(0) + .unwrap() + .clone(); + assert_eq!(transfer1.block_number, 28056721); + let transfer2 = storage + .get_transfer_by_tx_hash_and_log_index(&chain, TX_HASH.to_string(), LOG_INDEX) + .await + .unwrap() + .unwrap(); + assert_eq!(transfer2.block_number, 28056726); + let transfer_from = storage.get_transfers_from_block(chain, 28056721).await.unwrap(); + assert_eq!(transfer_from.len(), 3); +}); + +cross_test!(test_last_transfer_block, { + let chain = Chain::Bsc; + let storage = init_nft_history_storage(&chain).await; + let transfers = nft_transfer_history(); + storage.add_transfers_to_history(chain, transfers).await.unwrap(); + + let last_block = NftTransferHistoryStorageOps::get_last_block_number(&storage, &chain) + .await + .unwrap() + .unwrap(); + assert_eq!(last_block, 28056726); +}); + +cross_test!(test_transfer_history, { + let chain = Chain::Bsc; + let storage = init_nft_history_storage(&chain).await; + let transfers = nft_transfer_history(); + storage.add_transfers_to_history(chain, transfers).await.unwrap(); + + let transfer_history = storage + .get_transfer_history(vec![chain], false, 1, Some(NonZeroUsize::new(3).unwrap()), None) + .await + .unwrap(); + assert_eq!(transfer_history.transfer_history.len(), 1); + let transfer = transfer_history.transfer_history.get(0).unwrap(); + assert_eq!(transfer.block_number, 28056721); + assert_eq!(transfer_history.skipped, 2); + assert_eq!(transfer_history.total, 4); +}); + +cross_test!(test_transfer_history_filters, { + let chain = Chain::Bsc; + let storage = init_nft_history_storage(&chain).await; + let transfers = nft_transfer_history(); + storage.add_transfers_to_history(chain, transfers).await.unwrap(); + + let filters = NftTransferHistoryFilters { + receive: true, + send: false, + from_date: None, + to_date: None, + exclude_spam: false, + exclude_phishing: false, + }; + + let filters1 = NftTransferHistoryFilters { + receive: false, + send: false, + from_date: None, + to_date: Some(1677166110), + exclude_spam: false, + exclude_phishing: false, + }; + + let filters2 = NftTransferHistoryFilters { + receive: false, + send: false, + from_date: Some(1677166110), + to_date: Some(1683627417), + exclude_spam: false, + exclude_phishing: false, + }; + + let transfer_history = storage + .get_transfer_history(vec![chain], true, 1, None, Some(filters)) + .await + .unwrap(); + assert_eq!(transfer_history.transfer_history.len(), 4); + let transfer = transfer_history.transfer_history.get(0).unwrap(); + assert_eq!(transfer.block_number, 28056726); + + let transfer_history1 = storage + .get_transfer_history(vec![chain], true, 1, None, Some(filters1)) + .await + .unwrap(); + assert_eq!(transfer_history1.transfer_history.len(), 1); + let transfer1 = transfer_history1.transfer_history.get(0).unwrap(); + assert_eq!(transfer1.block_number, 25919780); + + let transfer_history2 = storage + .get_transfer_history(vec![chain], true, 1, None, Some(filters2)) + .await + .unwrap(); + assert_eq!(transfer_history2.transfer_history.len(), 2); + let transfer_0 = transfer_history2.transfer_history.get(0).unwrap(); + assert_eq!(transfer_0.block_number, 28056721); + let transfer_1 = transfer_history2.transfer_history.get(1).unwrap(); + assert_eq!(transfer_1.block_number, 25919780); +}); + +cross_test!(test_get_update_transfer_meta, { + let chain = Chain::Bsc; + let storage = init_nft_history_storage(&chain).await; + let transfers = nft_transfer_history(); + storage.add_transfers_to_history(chain, transfers).await.unwrap(); + + let vec_token_add_id = storage.get_transfers_with_empty_meta(chain).await.unwrap(); + assert_eq!(vec_token_add_id.len(), 3); + + let token_add = "0x5c7d6712dfaf0cb079d48981781c8705e8417ca0".to_string(); + let transfer_meta = TransferMeta { + token_address: token_add.clone(), + token_id: Default::default(), + token_uri: None, + token_domain: None, + collection_name: None, + image_url: None, + image_domain: None, + token_name: Some("Tiki box".to_string()), + }; + storage + .update_transfers_meta_by_token_addr_id(&chain, transfer_meta, true) + .await + .unwrap(); + let transfer_upd = storage + .get_transfers_by_token_addr_id(chain, token_add, Default::default()) + .await + .unwrap(); + let transfer_upd = transfer_upd.get(0).unwrap(); + assert_eq!(transfer_upd.token_name, Some("Tiki box".to_string())); + assert!(transfer_upd.common.possible_spam); +}); + +cross_test!(test_update_transfer_spam_by_token_address, { + let chain = Chain::Bsc; + let storage = init_nft_history_storage(&chain).await; + let transfers = nft_transfer_history(); + storage.add_transfers_to_history(chain, transfers).await.unwrap(); + + storage + .update_transfer_spam_by_token_address(&chain, TOKEN_ADD.to_string(), true) + .await + .unwrap(); + let transfers = storage + .get_transfers_by_token_address(chain, TOKEN_ADD.to_string()) + .await + .unwrap(); + for transfers in transfers { + assert!(transfers.common.possible_spam); + } +}); + +cross_test!(test_get_token_addresses, { + let chain = Chain::Bsc; + let storage = init_nft_history_storage(&chain).await; + let transfers = nft_transfer_history(); + storage.add_transfers_to_history(chain, transfers).await.unwrap(); + + let token_addresses = storage.get_token_addresses(chain).await.unwrap(); + assert_eq!(token_addresses.len(), 2); +}); + +cross_test!(test_exclude_transfer_spam, { + let chain = Chain::Bsc; + let storage = init_nft_history_storage(&chain).await; + let transfers = nft_transfer_history(); + storage.add_transfers_to_history(chain, transfers).await.unwrap(); + + let filters = NftTransferHistoryFilters { + receive: true, + send: true, + from_date: None, + to_date: None, + exclude_spam: true, + exclude_phishing: false, + }; + let transfer_history = storage + .get_transfer_history(vec![chain], true, 1, None, Some(filters)) + .await + .unwrap(); + assert_eq!(transfer_history.transfer_history.len(), 3); +}); + +cross_test!(test_get_domains, { + let chain = Chain::Bsc; + let storage = init_nft_history_storage(&chain).await; + let transfers = nft_transfer_history(); + storage.add_transfers_to_history(chain, transfers).await.unwrap(); + + let domains = storage.get_domains(&chain).await.unwrap(); + assert_eq!(2, domains.len()); + assert!(domains.contains("tikimetadata.s3.amazonaws.com")); + assert!(domains.contains("public.nftstatic.com")); +}); + +cross_test!(test_update_transfer_phishing_by_domain, { + let chain = Chain::Bsc; + let storage = init_nft_history_storage(&chain).await; + let transfers = nft_transfer_history(); + storage.add_transfers_to_history(chain, transfers).await.unwrap(); + + let domains = vec![ + "tikimetadata.s3.amazonaws.com".to_string(), + "public.nftstatic.com".to_string(), + ]; + for domain in domains.into_iter() { storage - .update_transfer_spam_by_token_address(&chain, TOKEN_ADD.to_string(), true) + .update_transfer_phishing_by_domain(&chain, domain, true) .await .unwrap(); - let transfers = storage - .get_transfers_by_token_address(chain, TOKEN_ADD.to_string()) - .await - .unwrap(); - for transfers in transfers { - assert!(transfers.common.possible_spam); - } - }); - - cross_test!(test_get_token_addresses, { - let chain = Chain::Bsc; - let storage = init_nft_history_storage(&chain).await; - let transfers = nft_transfer_history(); - storage.add_transfers_to_history(chain, transfers).await.unwrap(); - - let token_addresses = storage.get_token_addresses(chain).await.unwrap(); - assert_eq!(token_addresses.len(), 2); - }); - - cross_test!(test_exclude_transfer_spam, { - let chain = Chain::Bsc; - let storage = init_nft_history_storage(&chain).await; - let transfers = nft_transfer_history(); - storage.add_transfers_to_history(chain, transfers).await.unwrap(); - - let filters = NftTransferHistoryFilters { - receive: true, - send: true, - from_date: None, - to_date: None, - exclude_spam: true, - exclude_phishing: false, - }; - let transfer_history = storage - .get_transfer_history(vec![chain], true, 1, None, Some(filters)) - .await - .unwrap(); - assert_eq!(transfer_history.transfer_history.len(), 3); - }); - - cross_test!(test_get_domains, { - let chain = Chain::Bsc; - let storage = init_nft_history_storage(&chain).await; - let transfers = nft_transfer_history(); - storage.add_transfers_to_history(chain, transfers).await.unwrap(); - - let domains = storage.get_domains(&chain).await.unwrap(); - assert_eq!(2, domains.len()); - assert!(domains.contains("tikimetadata.s3.amazonaws.com")); - assert!(domains.contains("public.nftstatic.com")); - }); - - cross_test!(test_update_transfer_phishing_by_domain, { - let chain = Chain::Bsc; - let storage = init_nft_history_storage(&chain).await; - let transfers = nft_transfer_history(); - storage.add_transfers_to_history(chain, transfers).await.unwrap(); - - let domains = vec![ - "tikimetadata.s3.amazonaws.com".to_string(), - "public.nftstatic.com".to_string(), - ]; - for domain in domains.into_iter() { - storage - .update_transfer_phishing_by_domain(&chain, domain, true) - .await - .unwrap(); - } - let transfers = storage - .get_transfer_history(vec![chain], true, 1, None, None) - .await - .unwrap() - .transfer_history; - for transfer in transfers.into_iter() { - assert!(transfer.possible_phishing); - } - }); -} + } + let transfers = storage + .get_transfer_history(vec![chain], true, 1, None, None) + .await + .unwrap() + .transfer_history; + for transfer in transfers.into_iter() { + assert!(transfer.possible_phishing); + } +}); diff --git a/mm2src/coins/nft/storage/db_test_helpers.rs b/mm2src/coins/nft/storage/db_test_helpers.rs index 69ecae1f76..497b98ae00 100644 --- a/mm2src/coins/nft/storage/db_test_helpers.rs +++ b/mm2src/coins/nft/storage/db_test_helpers.rs @@ -1,18 +1,11 @@ -use crate::eth::eth_addr_to_hex; -use crate::nft::nft_structs::{Chain, ContractType, Nft, NftCommon, NftListFilters, NftTransferCommon, - NftTransferHistory, NftTransferHistoryFilters, TransferMeta, TransferStatus, UriMeta}; -use crate::nft::storage::{NftListStorageOps, NftStorageBuilder, NftTransferHistoryStorageOps, RemoveNftResult}; +use crate::nft::nft_structs::{Chain, ContractType, Nft, NftCommon, NftTransferCommon, NftTransferHistory, + TransferStatus, UriMeta}; +use crate::nft::storage::{NftListStorageOps, NftStorageBuilder, NftTransferHistoryStorageOps}; use ethereum_types::Address; use mm2_number::BigDecimal; use mm2_test_helpers::for_tests::mm_ctx_with_custom_db; -use std::num::NonZeroUsize; use std::str::FromStr; -const TOKEN_ADD: &str = "0xfd913a305d70a60aac4faac70c739563738e1f81"; -const TOKEN_ID: &str = "214300044414"; -const TX_HASH: &str = "0x1e9f04e9b571b283bde02c98c2a97da39b2bb665b57c1f2b0b733f9b681debbe"; -const LOG_INDEX: u32 = 495; - pub(crate) fn nft() -> Nft { Nft { common: NftCommon { @@ -368,439 +361,3 @@ pub(crate) async fn init_nft_history_storage(chain: &Chain) -> impl NftListStora assert!(is_initialized); storage } - -pub(crate) async fn test_add_get_nfts_impl() { - let chain = Chain::Bsc; - let storage = init_nft_list_storage(&chain).await; - let nft_list = nft_list(); - storage.add_nfts_to_list(chain, nft_list, 28056726).await.unwrap(); - - let token_id = BigDecimal::from_str(TOKEN_ID).unwrap(); - let nft = storage - .get_nft(&chain, TOKEN_ADD.to_string(), token_id) - .await - .unwrap() - .unwrap(); - assert_eq!(nft.block_number, 28056721); -} - -pub(crate) async fn test_last_nft_block_impl() { - let chain = Chain::Bsc; - let storage = init_nft_list_storage(&chain).await; - let nft_list = nft_list(); - storage.add_nfts_to_list(chain, nft_list, 28056726).await.unwrap(); - - let last_block = NftListStorageOps::get_last_block_number(&storage, &chain) - .await - .unwrap() - .unwrap(); - assert_eq!(last_block, 28056726); -} - -pub(crate) async fn test_nft_list_impl() { - let chain = Chain::Bsc; - let storage = init_nft_list_storage(&chain).await; - let nft_list = nft_list(); - storage.add_nfts_to_list(chain, nft_list, 28056726).await.unwrap(); - - let nft_list = storage - .get_nft_list(vec![chain], false, 1, Some(NonZeroUsize::new(3).unwrap()), None) - .await - .unwrap(); - assert_eq!(nft_list.nfts.len(), 1); - let nft = nft_list.nfts.get(0).unwrap(); - assert_eq!(nft.block_number, 28056721); - assert_eq!(nft_list.skipped, 2); - assert_eq!(nft_list.total, 4); -} - -pub(crate) async fn test_remove_nft_impl() { - let chain = Chain::Bsc; - let storage = init_nft_list_storage(&chain).await; - let nft_list = nft_list(); - storage.add_nfts_to_list(chain, nft_list, 28056726).await.unwrap(); - - let token_id = BigDecimal::from_str(TOKEN_ID).unwrap(); - let remove_rslt = storage - .remove_nft_from_list(&chain, TOKEN_ADD.to_string(), token_id, 28056800) - .await - .unwrap(); - assert_eq!(remove_rslt, RemoveNftResult::NftRemoved); - let list_len = storage - .get_nft_list(vec![chain], true, 1, None, None) - .await - .unwrap() - .nfts - .len(); - assert_eq!(list_len, 3); - let last_scanned_block = storage.get_last_scanned_block(&chain).await.unwrap().unwrap(); - assert_eq!(last_scanned_block, 28056800); -} - -pub(crate) async fn test_nft_amount_impl() { - let chain = Chain::Bsc; - let storage = init_nft_list_storage(&chain).await; - let mut nft = nft(); - storage - .add_nfts_to_list(chain, vec![nft.clone()], 25919780) - .await - .unwrap(); - - nft.common.amount -= BigDecimal::from(1); - storage.update_nft_amount(&chain, nft.clone(), 25919800).await.unwrap(); - let amount = storage - .get_nft_amount( - &chain, - eth_addr_to_hex(&nft.common.token_address), - nft.common.token_id.clone(), - ) - .await - .unwrap() - .unwrap(); - assert_eq!(amount, "1"); - let last_scanned_block = storage.get_last_scanned_block(&chain).await.unwrap().unwrap(); - assert_eq!(last_scanned_block, 25919800); - - nft.common.amount += BigDecimal::from(1); - nft.block_number = 25919900; - storage - .update_nft_amount_and_block_number(&chain, nft.clone()) - .await - .unwrap(); - let amount = storage - .get_nft_amount(&chain, eth_addr_to_hex(&nft.common.token_address), nft.common.token_id) - .await - .unwrap() - .unwrap(); - assert_eq!(amount, "2"); - let last_scanned_block = storage.get_last_scanned_block(&chain).await.unwrap().unwrap(); - assert_eq!(last_scanned_block, 25919900); -} - -pub(crate) async fn test_refresh_metadata_impl() { - let chain = Chain::Bsc; - let storage = init_nft_list_storage(&chain).await; - let new_symbol = "NEW_SYMBOL"; - let mut nft = nft(); - storage - .add_nfts_to_list(chain, vec![nft.clone()], 25919780) - .await - .unwrap(); - nft.common.symbol = Some(new_symbol.to_string()); - drop_mutability!(nft); - let token_add = eth_addr_to_hex(&nft.common.token_address); - let token_id = nft.common.token_id.clone(); - storage.refresh_nft_metadata(&chain, nft).await.unwrap(); - let nft_upd = storage.get_nft(&chain, token_add, token_id).await.unwrap().unwrap(); - assert_eq!(new_symbol.to_string(), nft_upd.common.symbol.unwrap()); -} - -pub(crate) async fn test_update_nft_spam_by_token_address_impl() { - let chain = Chain::Bsc; - let storage = init_nft_list_storage(&chain).await; - let nft_list = nft_list(); - storage.add_nfts_to_list(chain, nft_list, 28056726).await.unwrap(); - - storage - .update_nft_spam_by_token_address(&chain, TOKEN_ADD.to_string(), true) - .await - .unwrap(); - let nfts = storage - .get_nfts_by_token_address(chain, TOKEN_ADD.to_string()) - .await - .unwrap(); - for nft in nfts { - assert!(nft.common.possible_spam); - } -} - -pub(crate) async fn test_exclude_nft_spam_impl() { - let chain = Chain::Bsc; - let storage = init_nft_list_storage(&chain).await; - let nft_list = nft_list(); - storage.add_nfts_to_list(chain, nft_list, 28056726).await.unwrap(); - - let filters = NftListFilters { - exclude_spam: true, - exclude_phishing: false, - }; - let nft_list = storage - .get_nft_list(vec![chain], true, 1, None, Some(filters)) - .await - .unwrap(); - assert_eq!(nft_list.nfts.len(), 3); -} - -pub(crate) async fn test_get_animation_external_domains_impl() { - let chain = Chain::Bsc; - let storage = init_nft_list_storage(&chain).await; - let nft_list = nft_list(); - storage.add_nfts_to_list(chain, nft_list, 28056726).await.unwrap(); - - let domains = storage.get_animation_external_domains(&chain).await.unwrap(); - assert_eq!(2, domains.len()); - assert!(domains.contains("tikimetadata.s3.amazonaws.com")); - assert!(domains.contains("public.nftstatic.com")); -} - -pub(crate) async fn test_update_nft_phishing_by_domain_impl() { - let chain = Chain::Bsc; - let storage = init_nft_list_storage(&chain).await; - let nft_list = nft_list(); - storage.add_nfts_to_list(chain, nft_list, 28056726).await.unwrap(); - - let domains = vec![ - "tikimetadata.s3.amazonaws.com".to_string(), - "public.nftstatic.com".to_string(), - ]; - for domain in domains.into_iter() { - storage - .update_nft_phishing_by_domain(&chain, domain, true) - .await - .unwrap(); - } - let nfts = storage - .get_nft_list(vec![chain], true, 1, None, None) - .await - .unwrap() - .nfts; - for nft in nfts.into_iter() { - assert!(nft.possible_phishing); - } -} - -pub(crate) async fn test_add_get_transfers_impl() { - let chain = Chain::Bsc; - let storage = init_nft_history_storage(&chain).await; - let transfers = nft_transfer_history(); - storage.add_transfers_to_history(chain, transfers).await.unwrap(); - - let token_id = BigDecimal::from_str(TOKEN_ID).unwrap(); - let transfer1 = storage - .get_transfers_by_token_addr_id(chain, TOKEN_ADD.to_string(), token_id) - .await - .unwrap() - .get(0) - .unwrap() - .clone(); - assert_eq!(transfer1.block_number, 28056721); - let transfer2 = storage - .get_transfer_by_tx_hash_and_log_index(&chain, TX_HASH.to_string(), LOG_INDEX) - .await - .unwrap() - .unwrap(); - assert_eq!(transfer2.block_number, 28056726); - let transfer_from = storage.get_transfers_from_block(chain, 28056721).await.unwrap(); - assert_eq!(transfer_from.len(), 3); -} - -pub(crate) async fn test_last_transfer_block_impl() { - let chain = Chain::Bsc; - let storage = init_nft_history_storage(&chain).await; - let transfers = nft_transfer_history(); - storage.add_transfers_to_history(chain, transfers).await.unwrap(); - - let last_block = NftTransferHistoryStorageOps::get_last_block_number(&storage, &chain) - .await - .unwrap() - .unwrap(); - assert_eq!(last_block, 28056726); -} - -pub(crate) async fn test_transfer_history_impl() { - let chain = Chain::Bsc; - let storage = init_nft_history_storage(&chain).await; - let transfers = nft_transfer_history(); - storage.add_transfers_to_history(chain, transfers).await.unwrap(); - - let transfer_history = storage - .get_transfer_history(vec![chain], false, 1, Some(NonZeroUsize::new(3).unwrap()), None) - .await - .unwrap(); - assert_eq!(transfer_history.transfer_history.len(), 1); - let transfer = transfer_history.transfer_history.get(0).unwrap(); - assert_eq!(transfer.block_number, 28056721); - assert_eq!(transfer_history.skipped, 2); - assert_eq!(transfer_history.total, 4); -} - -pub(crate) async fn test_transfer_history_filters_impl() { - let chain = Chain::Bsc; - let storage = init_nft_history_storage(&chain).await; - let transfers = nft_transfer_history(); - storage.add_transfers_to_history(chain, transfers).await.unwrap(); - - let filters = NftTransferHistoryFilters { - receive: true, - send: false, - from_date: None, - to_date: None, - exclude_spam: false, - exclude_phishing: false, - }; - - let filters1 = NftTransferHistoryFilters { - receive: false, - send: false, - from_date: None, - to_date: Some(1677166110), - exclude_spam: false, - exclude_phishing: false, - }; - - let filters2 = NftTransferHistoryFilters { - receive: false, - send: false, - from_date: Some(1677166110), - to_date: Some(1683627417), - exclude_spam: false, - exclude_phishing: false, - }; - - let transfer_history = storage - .get_transfer_history(vec![chain], true, 1, None, Some(filters)) - .await - .unwrap(); - assert_eq!(transfer_history.transfer_history.len(), 4); - let transfer = transfer_history.transfer_history.get(0).unwrap(); - assert_eq!(transfer.block_number, 28056726); - - let transfer_history1 = storage - .get_transfer_history(vec![chain], true, 1, None, Some(filters1)) - .await - .unwrap(); - assert_eq!(transfer_history1.transfer_history.len(), 1); - let transfer1 = transfer_history1.transfer_history.get(0).unwrap(); - assert_eq!(transfer1.block_number, 25919780); - - let transfer_history2 = storage - .get_transfer_history(vec![chain], true, 1, None, Some(filters2)) - .await - .unwrap(); - assert_eq!(transfer_history2.transfer_history.len(), 2); - let transfer_0 = transfer_history2.transfer_history.get(0).unwrap(); - assert_eq!(transfer_0.block_number, 28056721); - let transfer_1 = transfer_history2.transfer_history.get(1).unwrap(); - assert_eq!(transfer_1.block_number, 25919780); -} - -pub(crate) async fn test_get_update_transfer_meta_impl() { - let chain = Chain::Bsc; - let storage = init_nft_history_storage(&chain).await; - let transfers = nft_transfer_history(); - storage.add_transfers_to_history(chain, transfers).await.unwrap(); - - let vec_token_add_id = storage.get_transfers_with_empty_meta(chain).await.unwrap(); - assert_eq!(vec_token_add_id.len(), 3); - - let token_add = "0x5c7d6712dfaf0cb079d48981781c8705e8417ca0".to_string(); - let transfer_meta = TransferMeta { - token_address: token_add.clone(), - token_id: Default::default(), - token_uri: None, - token_domain: None, - collection_name: None, - image_url: None, - image_domain: None, - token_name: Some("Tiki box".to_string()), - }; - storage - .update_transfers_meta_by_token_addr_id(&chain, transfer_meta, true) - .await - .unwrap(); - let transfer_upd = storage - .get_transfers_by_token_addr_id(chain, token_add, Default::default()) - .await - .unwrap(); - let transfer_upd = transfer_upd.get(0).unwrap(); - assert_eq!(transfer_upd.token_name, Some("Tiki box".to_string())); - assert!(transfer_upd.common.possible_spam); -} - -pub(crate) async fn test_update_transfer_spam_by_token_address_impl() { - let chain = Chain::Bsc; - let storage = init_nft_history_storage(&chain).await; - let transfers = nft_transfer_history(); - storage.add_transfers_to_history(chain, transfers).await.unwrap(); - - storage - .update_transfer_spam_by_token_address(&chain, TOKEN_ADD.to_string(), true) - .await - .unwrap(); - let transfers = storage - .get_transfers_by_token_address(chain, TOKEN_ADD.to_string()) - .await - .unwrap(); - for transfers in transfers { - assert!(transfers.common.possible_spam); - } -} - -pub(crate) async fn test_get_token_addresses_impl() { - let chain = Chain::Bsc; - let storage = init_nft_history_storage(&chain).await; - let transfers = nft_transfer_history(); - storage.add_transfers_to_history(chain, transfers).await.unwrap(); - - let token_addresses = storage.get_token_addresses(chain).await.unwrap(); - assert_eq!(token_addresses.len(), 2); -} - -pub(crate) async fn test_exclude_transfer_spam_impl() { - let chain = Chain::Bsc; - let storage = init_nft_history_storage(&chain).await; - let transfers = nft_transfer_history(); - storage.add_transfers_to_history(chain, transfers).await.unwrap(); - - let filters = NftTransferHistoryFilters { - receive: true, - send: true, - from_date: None, - to_date: None, - exclude_spam: true, - exclude_phishing: false, - }; - let transfer_history = storage - .get_transfer_history(vec![chain], true, 1, None, Some(filters)) - .await - .unwrap(); - assert_eq!(transfer_history.transfer_history.len(), 3); -} - -pub(crate) async fn test_get_domains_impl() { - let chain = Chain::Bsc; - let storage = init_nft_history_storage(&chain).await; - let transfers = nft_transfer_history(); - storage.add_transfers_to_history(chain, transfers).await.unwrap(); - - let domains = storage.get_domains(&chain).await.unwrap(); - assert_eq!(2, domains.len()); - assert!(domains.contains("tikimetadata.s3.amazonaws.com")); - assert!(domains.contains("public.nftstatic.com")); -} - -pub(crate) async fn test_update_transfer_phishing_by_domain_impl() { - let chain = Chain::Bsc; - let storage = init_nft_history_storage(&chain).await; - let transfers = nft_transfer_history(); - storage.add_transfers_to_history(chain, transfers).await.unwrap(); - - let domains = vec![ - "tikimetadata.s3.amazonaws.com".to_string(), - "public.nftstatic.com".to_string(), - ]; - for domain in domains.into_iter() { - storage - .update_transfer_phishing_by_domain(&chain, domain, true) - .await - .unwrap(); - } - let transfers = storage - .get_transfer_history(vec![chain], true, 1, None, None) - .await - .unwrap() - .transfer_history; - for transfer in transfers.into_iter() { - assert!(transfer.possible_phishing); - } -} From bae45b8fd0b0b62e6dbbb93312a19eed926a684c Mon Sep 17 00:00:00 2001 From: laruh Date: Sun, 24 Sep 2023 23:36:59 +0700 Subject: [PATCH 33/39] protect_from_nft_spam_links, ignore get meta from moralis error --- mm2src/coins/nft.rs | 210 +++++++++++++++++++++----------- mm2src/coins/nft/nft_structs.rs | 46 ++++++- 2 files changed, 184 insertions(+), 72 deletions(-) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index 24faf81cb0..62a588fea2 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -16,9 +16,9 @@ use nft_structs::{Chain, ContractType, ConvertChain, Nft, NftFromMoralis, NftLis use crate::eth::{eth_addr_to_hex, get_eth_address, withdraw_erc1155, withdraw_erc721}; use crate::nft::nft_errors::{MetaFromUrlError, ProtectFromSpamError, UpdateSpamPhishingError}; -use crate::nft::nft_structs::{MnemonicHQRes, NftCommon, NftCtx, NftTransferCommon, PhishingDomainReq, - PhishingDomainRes, RefreshMetadataReq, SpamContractReq, SpamContractRes, TransferMeta, - TransferStatus, UriMeta}; +use crate::nft::nft_structs::{build_nft_with_empty_meta, BuildNftFields, MnemonicHQRes, NftCommon, NftCtx, + NftTransferCommon, PhishingDomainReq, PhishingDomainRes, RefreshMetadataReq, + SpamContractReq, SpamContractRes, TransferMeta, TransferStatus, UriMeta}; use crate::nft::storage::{NftListStorageOps, NftStorageBuilder, NftTransferHistoryStorageOps}; use common::parse_rfc3339_to_timestamp; use crypto::StandardHDCoinAddress; @@ -423,15 +423,27 @@ pub async fn refresh_nft_metadata(ctx: MmArc, req: RefreshMetadataReq) -> MmResu let _lock = nft_ctx.guard.lock().await; let storage = NftStorageBuilder::new(&ctx).build()?; - let token_address_str = format!("{:#02x}", req.token_address); - let moralis_meta = get_moralis_metadata( + let token_address_str = eth_addr_to_hex(&req.token_address); + let moralis_meta = match get_moralis_metadata( token_address_str.clone(), req.token_id.clone(), &req.chain, &req.url, &req.url_antispam, ) - .await?; + .await + { + Ok(moralis_meta) => moralis_meta, + Err(_) => { + storage + .update_nft_spam_by_token_address(&req.chain, token_address_str.clone(), true) + .await?; + storage + .update_transfer_spam_by_token_address(&req.chain, token_address_str.clone(), true) + .await?; + return Ok(()); + }, + }; let mut nft_db = storage .get_nft(&req.chain, token_address_str.clone(), req.token_id.clone()) .await? @@ -473,10 +485,9 @@ async fn update_transfer_meta_using_nft(storage: &T, chain: &Chain, nft: &mut where T: NftListStorageOps + NftTransferHistoryStorageOps, { - let set_spam = protect_from_nft_spam_links(nft, false)?; let transfer_meta = TransferMeta::from(nft.clone()); storage - .update_transfers_meta_by_token_addr_id(chain, transfer_meta, set_spam) + .update_transfers_meta_by_token_addr_id(chain, transfer_meta, nft.common.possible_spam) .await?; Ok(()) } @@ -697,7 +708,13 @@ async fn get_moralis_nft_transfers( Ok(res_list) } -/// **Caution:** ERC-1155 token can have a total supply more than 1, which means there could be several owners +/// Implements request to the Moralis "Get NFT metadata" endpoint. +/// +/// [Moralis Documentation Link](https://docs.moralis.io/web3-data-api/evm/reference/get-nft-metadata) +/// +/// **Caution:** +/// +/// ERC-1155 token can have a total supply more than 1, which means there could be several owners /// of the same token. `get_nft_metadata` returns NFTs info with the most recent owner. /// **Dont** use this function to get specific info about owner address, amount etc, you will get info not related to my_address. async fn get_moralis_metadata( @@ -725,7 +742,8 @@ async fn get_moralis_metadata( Some(contract_type) => contract_type, None => return MmError::err(GetNftInfoError::ContractTypeIsNull), }; - let nft_metadata = build_nft_from_moralis(*chain, nft_moralis, contract_type, url_antispam).await; + let mut nft_metadata = build_nft_from_moralis(*chain, nft_moralis, contract_type, url_antispam).await; + protect_from_nft_spam_links(&mut nft_metadata, false)?; Ok(nft_metadata) } @@ -915,21 +933,47 @@ async fn handle_receive_erc721 { - let mut nft = get_moralis_metadata( + let nft = match get_moralis_metadata( eth_addr_to_hex(&transfer.common.token_address), - transfer.common.token_id, + transfer.common.token_id.clone(), chain, url, url_antispam, ) - .await?; - // sometimes moralis updates Get All NFTs (which also affects Get Metadata) later - // than History by Wallet update - nft.common.owner_of = - Address::from_str(my_address).map_to_mm(|e| UpdateNftError::InvalidHexString(e.to_string()))?; - nft.block_number = transfer.block_number; - protect_from_nft_spam_links(&mut nft, false)?; - drop_mutability!(nft); + .await + { + Ok(mut moralis_meta) => { + // sometimes moralis updates Get All NFTs (which also affects Get Metadata) later + // than History by Wallet update + moralis_meta.common.owner_of = + Address::from_str(my_address).map_to_mm(|e| UpdateNftError::InvalidHexString(e.to_string()))?; + moralis_meta.block_number = transfer.block_number; + moralis_meta + }, + Err(_) => { + storage + .update_nft_spam_by_token_address(chain, eth_addr_to_hex(&transfer.common.token_address), true) + .await?; + storage + .update_transfer_spam_by_token_address( + chain, + eth_addr_to_hex(&transfer.common.token_address), + true, + ) + .await?; + build_nft_with_empty_meta(BuildNftFields { + token_address: transfer.common.token_address, + token_id: transfer.common.token_id, + amount: transfer.common.amount, + owner_of: Address::from_str(my_address) + .map_to_mm(|e| UpdateNftError::InvalidHexString(e.to_string()))?, + contract_type: transfer.contract_type, + possible_spam: true, + chain: transfer.chain, + block_number: transfer.block_number, + }) + }, + }; storage .add_nfts_to_list(*chain, vec![nft.clone()], transfer.block_number) .await?; @@ -991,18 +1035,15 @@ async fn handle_receive_erc1155 MmResult<(), UpdateNftError> { + let token_address_str = eth_addr_to_hex(&transfer.common.token_address); let mut nft = match storage - .get_nft( - chain, - eth_addr_to_hex(&transfer.common.token_address), - transfer.common.token_id.clone(), - ) + .get_nft(chain, token_address_str.clone(), transfer.common.token_id.clone()) .await? { Some(mut nft_db) => { // if owner address == from address, then owner sent tokens to themself, // which means that the amount will not change. - if my_address != eth_addr_to_hex(&transfer.common.from_address) { + if my_address != token_address_str { nft_db.common.amount += transfer.common.amount; } nft_db.block_number = transfer.block_number; @@ -1014,49 +1055,70 @@ async fn handle_receive_erc1155 { - let moralis_meta = get_moralis_metadata( - eth_addr_to_hex(&transfer.common.token_address), + let nft = match get_moralis_metadata( + token_address_str.clone(), transfer.common.token_id.clone(), chain, url, url_antispam, ) - .await?; - let token_uri = check_moralis_ipfs_bafy(moralis_meta.common.token_uri.as_deref()); - let token_domain = get_domain_from_url(token_uri.as_deref()); - let uri_meta = get_uri_meta( - token_uri.as_deref(), - moralis_meta.common.metadata.as_deref(), - url_antispam, - ) - .await; - let mut nft = Nft { - common: NftCommon { - token_address: moralis_meta.common.token_address, - token_id: moralis_meta.common.token_id, - amount: transfer.common.amount, - owner_of: Address::from_str(my_address) - .map_to_mm(|e| UpdateNftError::InvalidHexString(e.to_string()))?, - token_hash: moralis_meta.common.token_hash, - collection_name: moralis_meta.common.collection_name, - symbol: moralis_meta.common.symbol, - token_uri, - token_domain, - metadata: moralis_meta.common.metadata, - last_token_uri_sync: moralis_meta.common.last_token_uri_sync, - last_metadata_sync: moralis_meta.common.last_metadata_sync, - minter_address: moralis_meta.common.minter_address, - possible_spam: moralis_meta.common.possible_spam, + .await + { + Ok(moralis_meta) => { + let token_uri = check_moralis_ipfs_bafy(moralis_meta.common.token_uri.as_deref()); + let token_domain = get_domain_from_url(token_uri.as_deref()); + let uri_meta = get_uri_meta( + token_uri.as_deref(), + moralis_meta.common.metadata.as_deref(), + url_antispam, + ) + .await; + Nft { + common: NftCommon { + token_address: moralis_meta.common.token_address, + token_id: moralis_meta.common.token_id, + amount: transfer.common.amount, + owner_of: Address::from_str(my_address) + .map_to_mm(|e| UpdateNftError::InvalidHexString(e.to_string()))?, + token_hash: moralis_meta.common.token_hash, + collection_name: moralis_meta.common.collection_name, + symbol: moralis_meta.common.symbol, + token_uri, + token_domain, + metadata: moralis_meta.common.metadata, + last_token_uri_sync: moralis_meta.common.last_token_uri_sync, + last_metadata_sync: moralis_meta.common.last_metadata_sync, + minter_address: moralis_meta.common.minter_address, + possible_spam: moralis_meta.common.possible_spam, + }, + chain: *chain, + block_number_minted: moralis_meta.block_number_minted, + block_number: transfer.block_number, + contract_type: moralis_meta.contract_type, + possible_phishing: false, + uri_meta, + } + }, + Err(_) => { + storage + .update_nft_spam_by_token_address(chain, token_address_str.clone(), true) + .await?; + storage + .update_transfer_spam_by_token_address(chain, token_address_str, true) + .await?; + build_nft_with_empty_meta(BuildNftFields { + token_address: transfer.common.token_address, + token_id: transfer.common.token_id, + amount: transfer.common.amount, + owner_of: Address::from_str(my_address) + .map_to_mm(|e| UpdateNftError::InvalidHexString(e.to_string()))?, + contract_type: transfer.contract_type, + possible_spam: true, + chain: transfer.chain, + block_number: transfer.block_number, + }) }, - chain: *chain, - block_number_minted: moralis_meta.block_number_minted, - block_number: transfer.block_number, - contract_type: moralis_meta.contract_type, - possible_phishing: false, - uri_meta, }; - protect_from_nft_spam_links(&mut nft, false)?; - drop_mutability!(nft); storage .add_nfts_to_list(*chain, [nft.clone()], transfer.block_number) .await?; @@ -1132,14 +1194,26 @@ where { let nft_token_addr_id = storage.get_transfers_with_empty_meta(*chain).await?; for addr_id_pair in nft_token_addr_id.into_iter() { - let mut nft_meta = get_moralis_metadata( - addr_id_pair.token_address, + let mut nft_meta = match get_moralis_metadata( + addr_id_pair.token_address.clone(), addr_id_pair.token_id, chain, url, url_antispam, ) - .await?; + .await + { + Ok(nft_meta) => nft_meta, + Err(_) => { + storage + .update_nft_spam_by_token_address(chain, addr_id_pair.token_address.clone(), true) + .await?; + storage + .update_transfer_spam_by_token_address(chain, addr_id_pair.token_address, true) + .await?; + continue; + }, + }; update_transfer_meta_using_nft(storage, chain, &mut nft_meta).await?; } Ok(()) @@ -1191,16 +1265,16 @@ fn protect_from_history_spam_links( /// they must be just an arbitrary text, which represents NFT names. /// `symbol` also must be a text or sign that represents a symbol. /// This function also checks `metadata` field for spam. -fn protect_from_nft_spam_links(nft: &mut Nft, redact: bool) -> MmResult { +fn protect_from_nft_spam_links(nft: &mut Nft, redact: bool) -> MmResult<(), ProtectFromSpamError> { let collection_name_spam = process_text_for_spam_link(&mut nft.common.collection_name, redact)?; let symbol_spam = process_text_for_spam_link(&mut nft.common.symbol, redact)?; let token_name_spam = process_text_for_spam_link(&mut nft.uri_meta.token_name, redact)?; let meta_spam = process_metadata_for_spam_link(nft, redact)?; if collection_name_spam || symbol_spam || token_name_spam || meta_spam { - return Ok(true); + nft.common.possible_spam = true; } - Ok(false) + Ok(()) } /// The `process_metadata_for_spam_link` function checks and optionally redacts spam link in the `metadata` field of `Nft`. diff --git a/mm2src/coins/nft/nft_structs.rs b/mm2src/coins/nft/nft_structs.rs index d6d471dd78..a6a37cef17 100644 --- a/mm2src/coins/nft/nft_structs.rs +++ b/mm2src/coins/nft/nft_structs.rs @@ -62,9 +62,9 @@ pub struct NftListFilters { /// # Fields /// * `token_address`: The address of the NFT token. /// * `token_id`: The ID of the NFT token. -/// * `chain`: The blockchain chain where the NFT exists. +/// * `chain`: The blockchain where the NFT exists. /// * `protect_from_spam`: Indicates whether to check and redact potential spam. If set to true, -/// the internal function `protect_from_nft_spam` is utilized. +/// the internal function `protect_from_nft_spam` is utilized. #[derive(Debug, Deserialize)] pub struct NftMetadataReq { pub(crate) token_address: Address, @@ -78,7 +78,7 @@ pub struct NftMetadataReq { /// # Fields /// * `token_address`: The address of the NFT token whose metadata needs to be refreshed. /// * `token_id`: The ID of the NFT token. -/// * `chain`: The blockchain chain where the NFT exists. +/// * `chain`: The blockchain where the NFT exists. /// * `url`: URL to fetch the metadata. /// * `url_antispam`: URL used to validate if the fetched contract addresses are associated /// with spam contracts or if domain fields in the fetched metadata match known phishing domains. @@ -291,6 +291,44 @@ pub struct Nft { pub(crate) uri_meta: UriMeta, } +pub(crate) struct BuildNftFields { + pub(crate) token_address: Address, + pub(crate) token_id: BigDecimal, + pub(crate) amount: BigDecimal, + pub(crate) owner_of: Address, + pub(crate) contract_type: ContractType, + pub(crate) possible_spam: bool, + pub(crate) chain: Chain, + pub(crate) block_number: u64, +} + +pub(crate) fn build_nft_with_empty_meta(nft_fields: BuildNftFields) -> Nft { + Nft { + common: NftCommon { + token_address: nft_fields.token_address, + token_id: nft_fields.token_id, + amount: nft_fields.amount, + owner_of: nft_fields.owner_of, + token_hash: None, + collection_name: None, + symbol: None, + token_uri: None, + token_domain: None, + metadata: None, + last_token_uri_sync: None, + last_metadata_sync: None, + minter_address: None, + possible_spam: nft_fields.possible_spam, + }, + chain: nft_fields.chain, + block_number_minted: None, + block_number: nft_fields.block_number, + contract_type: nft_fields.contract_type, + possible_phishing: false, + uri_meta: Default::default(), + } +} + /// Represents an NFT structure specifically for deserialization from Moralis's JSON response. /// /// This structure is adapted to the specific format provided by Moralis's API. @@ -537,7 +575,7 @@ pub struct NftTransferHistoryFilters { /// Contains parameters required to update NFT transfer history and NFT list. /// # Fields -/// * `chains`: A list of blockchain chains for which the NFTs need to be updated. +/// * `chains`: A list of blockchains for which the NFTs need to be updated. /// * `url`: URL to fetch the NFT data. /// * `url_antispam`: URL used to validate if the fetched contract addresses are associated /// with spam contracts or if domain fields in the fetched metadata match known phishing domains. From dd228375df90c1c86929d78c7c72019190762dff Mon Sep 17 00:00:00 2001 From: laruh Date: Mon, 25 Sep 2023 17:46:01 +0700 Subject: [PATCH 34/39] test exclude phishing and spam --- mm2src/coins/nft/nft_tests.rs | 63 +++++++++++++++++++++ mm2src/coins/nft/storage/db_test_helpers.rs | 4 +- 2 files changed, 65 insertions(+), 2 deletions(-) diff --git a/mm2src/coins/nft/nft_tests.rs b/mm2src/coins/nft/nft_tests.rs index a7dc008d0e..51928b6744 100644 --- a/mm2src/coins/nft/nft_tests.rs +++ b/mm2src/coins/nft/nft_tests.rs @@ -380,6 +380,28 @@ cross_test!(test_update_nft_phishing_by_domain, { } }); +cross_test!(test_exclude_nft_phishing_spam, { + let chain = Chain::Bsc; + let storage = init_nft_list_storage(&chain).await; + let nft_list = nft_list(); + storage.add_nfts_to_list(chain, nft_list, 28056726).await.unwrap(); + + storage + .update_nft_phishing_by_domain(&chain, "tikimetadata.s3.amazonaws.com".to_string(), true) + .await + .unwrap(); + let filters = NftListFilters { + exclude_spam: true, + exclude_phishing: true, + }; + let nfts = storage + .get_nft_list(vec![chain], true, 1, None, Some(filters)) + .await + .unwrap() + .nfts; + assert_eq!(nfts.len(), 2); +}); + cross_test!(test_add_get_transfers, { let chain = Chain::Bsc; let storage = init_nft_history_storage(&chain).await; @@ -615,3 +637,44 @@ cross_test!(test_update_transfer_phishing_by_domain, { assert!(transfer.possible_phishing); } }); + +cross_test!(test_exclude_transfer_phishing_spam, { + let chain = Chain::Bsc; + let storage = init_nft_history_storage(&chain).await; + let transfers = nft_transfer_history(); + storage.add_transfers_to_history(chain, transfers).await.unwrap(); + + storage + .update_transfer_phishing_by_domain(&chain, "tikimetadata.s3.amazonaws.com".to_string(), true) + .await + .unwrap(); + let filters = NftTransferHistoryFilters { + receive: true, + send: true, + from_date: None, + to_date: None, + exclude_spam: false, + exclude_phishing: true, + }; + let transfers = storage + .get_transfer_history(vec![chain], true, 1, None, Some(filters)) + .await + .unwrap() + .transfer_history; + assert_eq!(transfers.len(), 2); + + let filters1 = NftTransferHistoryFilters { + receive: true, + send: true, + from_date: None, + to_date: None, + exclude_spam: true, + exclude_phishing: true, + }; + let transfers = storage + .get_transfer_history(vec![chain], true, 1, None, Some(filters1)) + .await + .unwrap() + .transfer_history; + assert_eq!(transfers.len(), 1); +}); diff --git a/mm2src/coins/nft/storage/db_test_helpers.rs b/mm2src/coins/nft/storage/db_test_helpers.rs index 497b98ae00..9d4e8bfe14 100644 --- a/mm2src/coins/nft/storage/db_test_helpers.rs +++ b/mm2src/coins/nft/storage/db_test_helpers.rs @@ -64,7 +64,7 @@ pub(crate) fn nft_list() -> Vec { last_token_uri_sync: Some("2023-02-07T17:10:08.402Z".to_string()), last_metadata_sync: Some("2023-02-07T17:10:16.858Z".to_string()), minter_address: Some("ERC1155 tokens don't have a single minter".to_string()), - possible_spam: true, + possible_spam: false, }, chain: Chain::Bsc, block_number_minted: Some(25465916), @@ -104,7 +104,7 @@ pub(crate) fn nft_list() -> Vec { last_token_uri_sync: Some("2023-02-16T16:35:52.392Z".to_string()), last_metadata_sync: Some("2023-02-16T16:36:04.283Z".to_string()), minter_address: Some("0xdbdeb0895f3681b87fb3654b5cf3e05546ba24a9".to_string()), - possible_spam: false, + possible_spam: true, }, chain: Chain::Bsc, block_number_minted: Some(25721963), From df19597fc02aaefc90e2bca2c99c9b9f1686a608 Mon Sep 17 00:00:00 2001 From: laruh Date: Tue, 26 Sep 2023 11:29:38 +0700 Subject: [PATCH 35/39] remove MnemonicHQ endpoint --- mm2src/coins/nft.rs | 57 ++++----------------------------- mm2src/coins/nft/nft_structs.rs | 9 ------ mm2src/coins/nft/nft_tests.rs | 15 +-------- 3 files changed, 8 insertions(+), 73 deletions(-) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index 62a588fea2..ac508fdf02 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -16,9 +16,9 @@ use nft_structs::{Chain, ContractType, ConvertChain, Nft, NftFromMoralis, NftLis use crate::eth::{eth_addr_to_hex, get_eth_address, withdraw_erc1155, withdraw_erc721}; use crate::nft::nft_errors::{MetaFromUrlError, ProtectFromSpamError, UpdateSpamPhishingError}; -use crate::nft::nft_structs::{build_nft_with_empty_meta, BuildNftFields, MnemonicHQRes, NftCommon, NftCtx, - NftTransferCommon, PhishingDomainReq, PhishingDomainRes, RefreshMetadataReq, - SpamContractReq, SpamContractRes, TransferMeta, TransferStatus, UriMeta}; +use crate::nft::nft_structs::{build_nft_with_empty_meta, BuildNftFields, NftCommon, NftCtx, NftTransferCommon, + PhishingDomainReq, PhishingDomainRes, RefreshMetadataReq, SpamContractReq, + SpamContractRes, TransferMeta, TransferStatus, UriMeta}; use crate::nft::storage::{NftListStorageOps, NftStorageBuilder, NftTransferHistoryStorageOps}; use common::parse_rfc3339_to_timestamp; use crypto::StandardHDCoinAddress; @@ -48,7 +48,6 @@ const MORALIS_FROM_BLOCK_QUERY_NAME: &str = "from_block"; const BLOCKLIST_ENDPOINT: &str = "api/blocklist"; const BLOCKLIST_CONTRACT: &str = "contract"; const BLOCKLIST_DOMAIN: &str = "domain"; -const BLOCKLIST_WALLET: &str = "wallet"; const BLOCKLIST_SCAN: &str = "scan"; /// `WithdrawNftResult` type represents the result of an NFT withdrawal operation. On success, it provides the details @@ -216,7 +215,7 @@ pub async fn update_nft(ctx: MmArc, req: UpdateNftReq) -> MmResult<(), UpdateNft let nft_list = cache_nfts_from_moralis(&ctx, &storage, chain, &req.url, &req.url_antispam).await?; update_meta_in_transfers(&storage, chain, nft_list).await?; update_transfers_with_empty_meta(&storage, chain, &req.url, &req.url_antispam).await?; - update_spam(&ctx, &storage, *chain, &req.url_antispam).await?; + update_spam(&storage, *chain, &req.url_antispam).await?; update_phishing(&storage, chain, &req.url_antispam).await?; continue; }, @@ -226,7 +225,7 @@ pub async fn update_nft(ctx: MmArc, req: UpdateNftReq) -> MmResult<(), UpdateNft let nft_list = cache_nfts_from_moralis(&ctx, &storage, chain, &req.url, &req.url_antispam).await?; update_meta_in_transfers(&storage, chain, nft_list).await?; update_transfers_with_empty_meta(&storage, chain, &req.url, &req.url_antispam).await?; - update_spam(&ctx, &storage, *chain, &req.url_antispam).await?; + update_spam(&storage, *chain, &req.url_antispam).await?; update_phishing(&storage, chain, &req.url_antispam).await?; continue; }, @@ -256,25 +255,17 @@ pub async fn update_nft(ctx: MmArc, req: UpdateNftReq) -> MmResult<(), UpdateNft ) .await?; update_transfers_with_empty_meta(&storage, chain, &req.url, &req.url_antispam).await?; - update_spam(&ctx, &storage, *chain, &req.url_antispam).await?; + update_spam(&storage, *chain, &req.url_antispam).await?; update_phishing(&storage, chain, &req.url_antispam).await?; } Ok(()) } /// `update_spam` function updates spam contracts info in NFT list and NFT transfers. -async fn update_spam( - ctx: &MmArc, - storage: &T, - chain: Chain, - url_antispam: &Url, -) -> MmResult<(), UpdateSpamPhishingError> +async fn update_spam(storage: &T, chain: Chain, url_antispam: &Url) -> MmResult<(), UpdateSpamPhishingError> where T: NftListStorageOps + NftTransferHistoryStorageOps, { - if chain == Chain::Eth || chain == Chain::Polygon { - return update_spam_nft_with_mnemonichq(ctx, storage, &chain, url_antispam).await; - } let token_addresses = storage.get_token_addresses(chain).await?; if !token_addresses.is_empty() { let addresses = token_addresses @@ -322,40 +313,6 @@ where Ok(()) } -/// Uses `/api/blocklist/wallet/{network}/{wallet_address}` endpoint to get **all spam contract addresses** of NFTs owned by the user. -/// Currently, only the `Eth` and `Polygon` networks are supported. -async fn update_spam_nft_with_mnemonichq( - ctx: &MmArc, - storage: &T, - chain: &Chain, - url_antispam: &Url, -) -> MmResult<(), UpdateSpamPhishingError> -where - T: NftListStorageOps + NftTransferHistoryStorageOps, -{ - let mut scan_wallet_uri = prepare_uri_for_blocklist_endpoint(url_antispam, BLOCKLIST_WALLET, &chain.to_string())?; - let req = MyAddressReq { - coin: chain.to_ticker(), - path_to_address: Default::default(), - }; - let my_address = get_my_address(ctx.clone(), req).await?.wallet_address.to_lowercase(); - scan_wallet_uri - .path_segments_mut() - .map_to_mm(|_| UpdateSpamPhishingError::Internal("Invalid URI".to_string()))? - .push(&my_address); - let response = send_request_to_uri(scan_wallet_uri.as_str()).await?; - let mnemonichq_res: MnemonicHQRes = serde_json::from_value(response)?; - for contract in mnemonichq_res.spam_contracts.iter() { - storage - .update_nft_spam_by_token_address(chain, eth_addr_to_hex(contract), true) - .await?; - storage - .update_transfer_spam_by_token_address(chain, eth_addr_to_hex(contract), true) - .await?; - } - Ok(()) -} - /// `send_spam_request` function sends request to antispam api to scan contract addresses for spam. async fn send_spam_request( chain: &Chain, diff --git a/mm2src/coins/nft/nft_structs.rs b/mm2src/coins/nft/nft_structs.rs index a6a37cef17..165e7cbd93 100644 --- a/mm2src/coins/nft/nft_structs.rs +++ b/mm2src/coins/nft/nft_structs.rs @@ -667,12 +667,3 @@ pub(crate) struct SpamContractRes { pub(crate) struct PhishingDomainRes { pub(crate) result: HashMap, } - -#[allow(dead_code)] -#[derive(Debug, Deserialize)] -pub(crate) struct MnemonicHQRes { - pub(crate) network: Chain, - pub(crate) address: Address, - pub(crate) result: String, - pub(crate) spam_contracts: Vec
, -} diff --git a/mm2src/coins/nft/nft_tests.rs b/mm2src/coins/nft/nft_tests.rs index 51928b6744..b8a89e714e 100644 --- a/mm2src/coins/nft/nft_tests.rs +++ b/mm2src/coins/nft/nft_tests.rs @@ -1,5 +1,5 @@ use crate::eth::eth_addr_to_hex; -use crate::nft::nft_structs::{Chain, MnemonicHQRes, NftFromMoralis, NftListFilters, NftTransferHistoryFilters, +use crate::nft::nft_structs::{Chain, NftFromMoralis, NftListFilters, NftTransferHistoryFilters, NftTransferHistoryFromMoralis, PhishingDomainReq, PhishingDomainRes, SpamContractReq, SpamContractRes, TransferMeta, UriMeta}; use crate::nft::storage::db_test_helpers::{init_nft_history_storage, init_nft_list_storage, nft, nft_list, @@ -128,19 +128,6 @@ cross_test!(test_moralis_requests, { assert_eq!(41237364, *nft_moralis.block_number_minted.unwrap()); }); -cross_test!(test_antispam_wallet_endpoint, { - let uri_mnemonichq = format!( - "{}/api/blocklist/wallet/eth/0x3eb4b12127EdC81A4d2fD49658db07005bcAd065", - BLOCKLIST_API_ENDPOINT - ); - let res_value = send_request_to_uri(uri_mnemonichq.as_str()).await.unwrap(); - let mnemonichq_res: MnemonicHQRes = serde_json::from_value(res_value).unwrap(); - assert!(mnemonichq_res - .spam_contracts - .contains(&Address::from_str("0x0ded8542fc8b2b4e781b96e99fee6406550c9b7c").unwrap())); - assert_eq!(Chain::Eth, mnemonichq_res.network); -}); - cross_test!(test_antispam_scan_endpoints, { let req_spam = SpamContractReq { network: Chain::Eth, From 3b8108b064bfc5f9bfe13838e5918bd15c39c75f Mon Sep 17 00:00:00 2001 From: laruh Date: Tue, 26 Sep 2023 12:31:01 +0700 Subject: [PATCH 36/39] polish handle_receive_erc --- mm2src/coins/nft.rs | 177 +++++++++++++++++++++----------------------- 1 file changed, 84 insertions(+), 93 deletions(-) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index ac508fdf02..c8f3f5818c 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -865,12 +865,9 @@ async fn handle_receive_erc721 MmResult<(), UpdateNftError> { - let mut nft = match storage - .get_nft( - chain, - eth_addr_to_hex(&transfer.common.token_address), - transfer.common.token_id.clone(), - ) + let token_address_str = eth_addr_to_hex(&transfer.common.token_address); + match storage + .get_nft(chain, token_address_str.clone(), transfer.common.token_id.clone()) .await? { Some(mut nft_db) => { @@ -882,16 +879,14 @@ async fn handle_receive_erc721 { - let nft = match get_moralis_metadata( - eth_addr_to_hex(&transfer.common.token_address), + let mut nft = match get_moralis_metadata( + token_address_str.clone(), transfer.common.token_id.clone(), chain, url, @@ -908,36 +903,15 @@ async fn handle_receive_erc721 { - storage - .update_nft_spam_by_token_address(chain, eth_addr_to_hex(&transfer.common.token_address), true) - .await?; - storage - .update_transfer_spam_by_token_address( - chain, - eth_addr_to_hex(&transfer.common.token_address), - true, - ) - .await?; - build_nft_with_empty_meta(BuildNftFields { - token_address: transfer.common.token_address, - token_id: transfer.common.token_id, - amount: transfer.common.amount, - owner_of: Address::from_str(my_address) - .map_to_mm(|e| UpdateNftError::InvalidHexString(e.to_string()))?, - contract_type: transfer.contract_type, - possible_spam: true, - chain: transfer.chain, - block_number: transfer.block_number, - }) + mark_as_spam_and_build_empty_meta(storage, chain, token_address_str, &transfer, my_address).await? }, }; storage .add_nfts_to_list(*chain, vec![nft.clone()], transfer.block_number) .await?; - nft + update_transfer_meta_using_nft(storage, chain, &mut nft).await?; }, - }; - update_transfer_meta_using_nft(storage, chain, &mut nft).await?; + } Ok(()) } @@ -946,15 +920,12 @@ async fn handle_send_erc1155 MmResult<(), UpdateNftError> { + let token_address_str = eth_addr_to_hex(&transfer.common.token_address); let mut nft_db = storage - .get_nft( - chain, - eth_addr_to_hex(&transfer.common.token_address), - transfer.common.token_id.clone(), - ) + .get_nft(chain, token_address_str.clone(), transfer.common.token_id.clone()) .await? .ok_or_else(|| UpdateNftError::TokenNotFoundInWallet { - token_address: eth_addr_to_hex(&transfer.common.token_address), + token_address: token_address_str.clone(), token_id: transfer.common.token_id.to_string(), })?; match nft_db.common.amount.cmp(&transfer.common.amount) { @@ -962,7 +933,7 @@ async fn handle_send_erc1155 { // if owner address == from address, then owner sent tokens to themself, // which means that the amount will not change. - if my_address != token_address_str { + if my_address != eth_addr_to_hex(&transfer.common.from_address) { nft_db.common.amount += transfer.common.amount; } nft_db.block_number = transfer.block_number; @@ -1022,58 +993,10 @@ async fn handle_receive_erc1155 { - let token_uri = check_moralis_ipfs_bafy(moralis_meta.common.token_uri.as_deref()); - let token_domain = get_domain_from_url(token_uri.as_deref()); - let uri_meta = get_uri_meta( - token_uri.as_deref(), - moralis_meta.common.metadata.as_deref(), - url_antispam, - ) - .await; - Nft { - common: NftCommon { - token_address: moralis_meta.common.token_address, - token_id: moralis_meta.common.token_id, - amount: transfer.common.amount, - owner_of: Address::from_str(my_address) - .map_to_mm(|e| UpdateNftError::InvalidHexString(e.to_string()))?, - token_hash: moralis_meta.common.token_hash, - collection_name: moralis_meta.common.collection_name, - symbol: moralis_meta.common.symbol, - token_uri, - token_domain, - metadata: moralis_meta.common.metadata, - last_token_uri_sync: moralis_meta.common.last_token_uri_sync, - last_metadata_sync: moralis_meta.common.last_metadata_sync, - minter_address: moralis_meta.common.minter_address, - possible_spam: moralis_meta.common.possible_spam, - }, - chain: *chain, - block_number_minted: moralis_meta.block_number_minted, - block_number: transfer.block_number, - contract_type: moralis_meta.contract_type, - possible_phishing: false, - uri_meta, - } + create_nft_from_moralis_metadata(moralis_meta, &transfer, my_address, chain, url_antispam).await? }, Err(_) => { - storage - .update_nft_spam_by_token_address(chain, token_address_str.clone(), true) - .await?; - storage - .update_transfer_spam_by_token_address(chain, token_address_str, true) - .await?; - build_nft_with_empty_meta(BuildNftFields { - token_address: transfer.common.token_address, - token_id: transfer.common.token_id, - amount: transfer.common.amount, - owner_of: Address::from_str(my_address) - .map_to_mm(|e| UpdateNftError::InvalidHexString(e.to_string()))?, - contract_type: transfer.contract_type, - possible_spam: true, - chain: transfer.chain, - block_number: transfer.block_number, - }) + mark_as_spam_and_build_empty_meta(storage, chain, token_address_str, &transfer, my_address).await? }, }; storage @@ -1086,6 +1009,74 @@ async fn handle_receive_erc1155 MmResult { + let token_uri = check_moralis_ipfs_bafy(moralis_meta.common.token_uri.as_deref()); + let token_domain = get_domain_from_url(token_uri.as_deref()); + let uri_meta = get_uri_meta( + token_uri.as_deref(), + moralis_meta.common.metadata.as_deref(), + url_antispam, + ) + .await; + let nft = Nft { + common: NftCommon { + token_address: moralis_meta.common.token_address, + token_id: moralis_meta.common.token_id, + amount: transfer.common.amount.clone(), + owner_of: Address::from_str(my_address).map_to_mm(|e| UpdateNftError::InvalidHexString(e.to_string()))?, + token_hash: moralis_meta.common.token_hash, + collection_name: moralis_meta.common.collection_name, + symbol: moralis_meta.common.symbol, + token_uri, + token_domain, + metadata: moralis_meta.common.metadata, + last_token_uri_sync: moralis_meta.common.last_token_uri_sync, + last_metadata_sync: moralis_meta.common.last_metadata_sync, + minter_address: moralis_meta.common.minter_address, + possible_spam: moralis_meta.common.possible_spam, + }, + chain: *chain, + block_number_minted: moralis_meta.block_number_minted, + block_number: transfer.block_number, + contract_type: moralis_meta.contract_type, + possible_phishing: false, + uri_meta, + }; + Ok(nft) +} + +async fn mark_as_spam_and_build_empty_meta( + storage: &T, + chain: &Chain, + token_address_str: String, + transfer: &NftTransferHistory, + my_address: &str, +) -> MmResult { + storage + .update_nft_spam_by_token_address(chain, token_address_str.clone(), true) + .await?; + storage + .update_transfer_spam_by_token_address(chain, token_address_str, true) + .await?; + + Ok(build_nft_with_empty_meta(BuildNftFields { + token_address: transfer.common.token_address, + token_id: transfer.common.token_id.clone(), + amount: transfer.common.amount.clone(), + owner_of: Address::from_str(my_address).map_to_mm(|e| UpdateNftError::InvalidHexString(e.to_string()))?, + contract_type: transfer.contract_type, + possible_spam: true, + chain: transfer.chain, + block_number: transfer.block_number, + })) +} + /// `find_wallet_nft_amount` function returns NFT amount of cached NFT. /// Note: in db **token_address** is kept in **lowercase**, because Moralis returns all addresses in lowercase. pub(crate) async fn find_wallet_nft_amount( From 92755444188ebe58419c6ea682ae2e7066284ccc Mon Sep 17 00:00:00 2001 From: laruh Date: Tue, 3 Oct 2023 18:49:08 +0700 Subject: [PATCH 37/39] url.path_segments_mut().ok(), move cross_test! to common.rs --- mm2src/coins/nft.rs | 5 +---- mm2src/coins/nft/nft_tests.rs | 13 +------------ mm2src/common/common.rs | 13 +++++++++++++ mm2src/mm2_event_stream/src/controller.rs | 13 +------------ 4 files changed, 16 insertions(+), 28 deletions(-) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index c8f3f5818c..c4543a0312 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -758,10 +758,7 @@ async fn get_uri_meta(token_uri: Option<&str>, metadata: Option<&str>, url_antis fn construct_camo_url_with_token(token_uri: &str, url_antispam: &Url) -> Option { let mut url = url_antispam.clone(); url.set_path("url/decode"); - match url.path_segments_mut() { - Ok(mut segments) => segments.push(hex::encode(token_uri).as_str()), - Err(_) => return None, - }; + url.path_segments_mut().ok()?.push(hex::encode(token_uri).as_str()); Some(url) } diff --git a/mm2src/coins/nft/nft_tests.rs b/mm2src/coins/nft/nft_tests.rs index b8a89e714e..ae10513987 100644 --- a/mm2src/coins/nft/nft_tests.rs +++ b/mm2src/coins/nft/nft_tests.rs @@ -7,6 +7,7 @@ use crate::nft::storage::db_test_helpers::{init_nft_history_storage, init_nft_li use crate::nft::storage::{NftListStorageOps, NftTransferHistoryStorageOps, RemoveNftResult}; use crate::nft::{check_moralis_ipfs_bafy, get_domain_from_url, process_metadata_for_spam_link, process_text_for_spam_link}; +use common::cross_test; use ethereum_types::Address; use mm2_net::transport::send_post_request_to_uri; use mm2_number::BigDecimal; @@ -30,18 +31,6 @@ common::cfg_wasm32! { use mm2_net::wasm_http::send_request_to_uri; } -macro_rules! cross_test { - ($test_name:ident, $test_code:block) => { - #[cfg(not(target_arch = "wasm32"))] - #[tokio::test(flavor = "multi_thread")] - async fn $test_name() { $test_code } - - #[cfg(target_arch = "wasm32")] - #[wasm_bindgen_test] - async fn $test_name() { $test_code } - }; -} - cross_test!(test_moralis_ipfs_bafy, { let uri = "https://ipfs.moralis.io:2053/ipfs/bafybeifnek24coy5xj5qabdwh24dlp5omq34nzgvazkfyxgnqms4eidsiq/1.json"; let res_uri = check_moralis_ipfs_bafy(Some(uri)); diff --git a/mm2src/common/common.rs b/mm2src/common/common.rs index 058661f42c..6fecf68462 100644 --- a/mm2src/common/common.rs +++ b/mm2src/common/common.rs @@ -103,6 +103,19 @@ macro_rules! some_or_return_ok_none { }; } +#[macro_export] +macro_rules! cross_test { + ($test_name:ident, $test_code:block) => { + #[cfg(not(target_arch = "wasm32"))] + #[tokio::test(flavor = "multi_thread")] + async fn $test_name() { $test_code } + + #[cfg(target_arch = "wasm32")] + #[wasm_bindgen_test] + async fn $test_name() { $test_code } + }; +} + #[macro_use] pub mod jsonrpc_client; #[macro_use] diff --git a/mm2src/mm2_event_stream/src/controller.rs b/mm2src/mm2_event_stream/src/controller.rs index 098c6e4bb7..72870308b4 100644 --- a/mm2src/mm2_event_stream/src/controller.rs +++ b/mm2src/mm2_event_stream/src/controller.rs @@ -103,24 +103,13 @@ impl GuardedReceiver { #[cfg(any(test, target_arch = "wasm32"))] mod tests { use super::*; + use common::cross_test; common::cfg_wasm32! { use wasm_bindgen_test::*; wasm_bindgen_test::wasm_bindgen_test_configure!(run_in_browser); } - macro_rules! cross_test { - ($test_name:ident, $test_code:block) => { - #[cfg(not(target_arch = "wasm32"))] - #[tokio::test(flavor = "multi_thread")] - async fn $test_name() { $test_code } - - #[cfg(target_arch = "wasm32")] - #[wasm_bindgen_test] - async fn $test_name() { $test_code } - }; - } - cross_test!(test_create_channel_and_broadcast, { let mut controller = Controller::new(); let mut guard_receiver = controller.create_channel(1); From 932364e2a64a27657fdb989fc14f2b94c3823a05 Mon Sep 17 00:00:00 2001 From: laruh Date: Thu, 5 Oct 2023 18:24:33 +0700 Subject: [PATCH 38/39] use const `DB_NAME` for trait `DbInstance` --- mm2src/coins/hd_wallet_storage/wasm_storage.rs | 3 +-- mm2src/coins/nft/storage/wasm/nft_idb.rs | 3 +-- mm2src/coins/tx_history_storage/wasm/tx_history_db.rs | 3 +-- .../wasm/indexeddb_block_header_storage.rs | 3 +-- mm2src/coins/z_coin/storage/blockdb/block_idb.rs | 3 +-- mm2src/coins/z_coin/storage/walletdb/wallet_idb.rs | 3 +-- mm2src/mm2_db/src/indexed_db/indexed_db.rs | 6 +++--- mm2src/mm2_gui_storage/src/account/storage/wasm_storage.rs | 3 +-- mm2src/mm2_main/src/lp_ordermatch/ordermatch_wasm_db.rs | 3 +-- mm2src/mm2_main/src/lp_swap/swap_wasm_db.rs | 3 +-- 10 files changed, 12 insertions(+), 21 deletions(-) diff --git a/mm2src/coins/hd_wallet_storage/wasm_storage.rs b/mm2src/coins/hd_wallet_storage/wasm_storage.rs index a9e11143e2..76ad67494f 100644 --- a/mm2src/coins/hd_wallet_storage/wasm_storage.rs +++ b/mm2src/coins/hd_wallet_storage/wasm_storage.rs @@ -10,7 +10,6 @@ use mm2_db::indexed_db::{DbIdentifier, DbInstance, DbLocked, DbTable, DbTransact TableSignature, WeakDb}; use mm2_err_handle::prelude::*; -const DB_NAME: &str = "hd_wallet"; const DB_VERSION: u32 = 1; /// An index of the `HDAccountTable` table that consists of the following properties: /// * coin - coin ticker @@ -142,7 +141,7 @@ pub struct HDWalletDb { #[async_trait] impl DbInstance for HDWalletDb { - fn db_name() -> &'static str { DB_NAME } + const DB_NAME: &'static str = "hd_wallet"; async fn init(db_id: DbIdentifier) -> InitDbResult { let inner = IndexedDbBuilder::new(db_id) diff --git a/mm2src/coins/nft/storage/wasm/nft_idb.rs b/mm2src/coins/nft/storage/wasm/nft_idb.rs index d388dd808a..054f1c058e 100644 --- a/mm2src/coins/nft/storage/wasm/nft_idb.rs +++ b/mm2src/coins/nft/storage/wasm/nft_idb.rs @@ -3,7 +3,6 @@ use async_trait::async_trait; use mm2_db::indexed_db::InitDbResult; use mm2_db::indexed_db::{DbIdentifier, DbInstance, DbLocked, IndexedDb, IndexedDbBuilder}; -const DB_NAME: &str = "nft_cache"; const DB_VERSION: u32 = 1; /// Represents a locked instance of the `NftCacheIDB` database. @@ -24,7 +23,7 @@ pub struct NftCacheIDB { #[async_trait] impl DbInstance for NftCacheIDB { - fn db_name() -> &'static str { DB_NAME } + const DB_NAME: &'static str = "nft_cache"; async fn init(db_id: DbIdentifier) -> InitDbResult { let inner = IndexedDbBuilder::new(db_id) diff --git a/mm2src/coins/tx_history_storage/wasm/tx_history_db.rs b/mm2src/coins/tx_history_storage/wasm/tx_history_db.rs index c88fd3defc..b646e7cefc 100644 --- a/mm2src/coins/tx_history_storage/wasm/tx_history_db.rs +++ b/mm2src/coins/tx_history_storage/wasm/tx_history_db.rs @@ -3,7 +3,6 @@ use crate::tx_history_storage::wasm::tx_history_storage_v2::{TxCacheTableV2, TxH use async_trait::async_trait; use mm2_db::indexed_db::{DbIdentifier, DbInstance, DbLocked, IndexedDb, IndexedDbBuilder, InitDbResult}; -const DB_NAME: &str = "tx_history"; const DB_VERSION: u32 = 1; pub type TxHistoryDbLocked<'a> = DbLocked<'a, TxHistoryDb>; @@ -14,7 +13,7 @@ pub struct TxHistoryDb { #[async_trait] impl DbInstance for TxHistoryDb { - fn db_name() -> &'static str { DB_NAME } + const DB_NAME: &'static str = "tx_history"; async fn init(db_id: DbIdentifier) -> InitDbResult { let inner = IndexedDbBuilder::new(db_id) diff --git a/mm2src/coins/utxo/utxo_block_header_storage/wasm/indexeddb_block_header_storage.rs b/mm2src/coins/utxo/utxo_block_header_storage/wasm/indexeddb_block_header_storage.rs index 13abdd4ed5..e958300e47 100644 --- a/mm2src/coins/utxo/utxo_block_header_storage/wasm/indexeddb_block_header_storage.rs +++ b/mm2src/coins/utxo/utxo_block_header_storage/wasm/indexeddb_block_header_storage.rs @@ -12,7 +12,6 @@ use serialization::Reader; use spv_validation::storage::{BlockHeaderStorageError, BlockHeaderStorageOps}; use std::collections::HashMap; -const DB_NAME: &str = "block_headers_cache"; const DB_VERSION: u32 = 1; pub type IDBBlockHeadersStorageRes = MmResult; @@ -24,7 +23,7 @@ pub struct IDBBlockHeadersInner { #[async_trait] impl DbInstance for IDBBlockHeadersInner { - fn db_name() -> &'static str { DB_NAME } + const DB_NAME: &'static str = "block_headers_cache"; async fn init(db_id: DbIdentifier) -> InitDbResult { let inner = IndexedDbBuilder::new(db_id) diff --git a/mm2src/coins/z_coin/storage/blockdb/block_idb.rs b/mm2src/coins/z_coin/storage/blockdb/block_idb.rs index e70cfce122..a1d4eda6d9 100644 --- a/mm2src/coins/z_coin/storage/blockdb/block_idb.rs +++ b/mm2src/coins/z_coin/storage/blockdb/block_idb.rs @@ -2,7 +2,6 @@ use async_trait::async_trait; use mm2_db::indexed_db::{BeBigUint, DbIdentifier, DbInstance, DbUpgrader, IndexedDb, IndexedDbBuilder, InitDbResult, OnUpgradeResult, TableSignature}; -const DB_NAME: &str = "z_compactblocks_cache"; const DB_VERSION: u32 = 1; #[derive(Clone, Debug, Deserialize, Serialize)] @@ -39,7 +38,7 @@ impl BlockDbInner { #[async_trait] impl DbInstance for BlockDbInner { - fn db_name() -> &'static str { DB_NAME } + const DB_NAME: &'static str = "z_compactblocks_cache"; async fn init(db_id: DbIdentifier) -> InitDbResult { let inner = IndexedDbBuilder::new(db_id) diff --git a/mm2src/coins/z_coin/storage/walletdb/wallet_idb.rs b/mm2src/coins/z_coin/storage/walletdb/wallet_idb.rs index 129097dbe6..e3abf591ce 100644 --- a/mm2src/coins/z_coin/storage/walletdb/wallet_idb.rs +++ b/mm2src/coins/z_coin/storage/walletdb/wallet_idb.rs @@ -2,7 +2,6 @@ use async_trait::async_trait; use mm2_db::indexed_db::{BeBigUint, DbIdentifier, DbInstance, DbUpgrader, IndexedDb, IndexedDbBuilder, InitDbResult, OnUpgradeResult, TableSignature}; -const DB_NAME: &str = "wallet_db_cache"; const DB_VERSION: u32 = 1; #[derive(Clone, Debug, Deserialize, Serialize)] @@ -224,7 +223,7 @@ impl WalletDbInner { #[async_trait] impl DbInstance for WalletDbInner { - fn db_name() -> &'static str { DB_NAME } + const DB_NAME: &'static str = "wallet_db_cache"; async fn init(db_id: DbIdentifier) -> InitDbResult { let inner = IndexedDbBuilder::new(db_id) diff --git a/mm2src/mm2_db/src/indexed_db/indexed_db.rs b/mm2src/mm2_db/src/indexed_db/indexed_db.rs index dd67ea195e..c1d56cd20d 100644 --- a/mm2src/mm2_db/src/indexed_db/indexed_db.rs +++ b/mm2src/mm2_db/src/indexed_db/indexed_db.rs @@ -65,11 +65,11 @@ pub trait TableSignature: DeserializeOwned + Serialize + 'static { fn on_upgrade_needed(upgrader: &DbUpgrader, old_version: u32, new_version: u32) -> OnUpgradeResult<()>; } -/// A trait representing the operations essential for initializing an IndexedDb instance. +/// Essential operations for initializing an IndexedDb instance. #[async_trait] pub trait DbInstance: Sized { /// Returns the static name of the database. - fn db_name() -> &'static str; + const DB_NAME: &'static str; /// Initialize the database with the provided identifier. /// This method ensures that the database is properly set up with the correct version @@ -94,7 +94,7 @@ impl DbIdentifier { DbIdentifier { namespace_id, wallet_rmd160, - db_name: Db::db_name(), + db_name: Db::DB_NAME, } } diff --git a/mm2src/mm2_gui_storage/src/account/storage/wasm_storage.rs b/mm2src/mm2_gui_storage/src/account/storage/wasm_storage.rs index 8392294c78..cbf9967d42 100644 --- a/mm2src/mm2_gui_storage/src/account/storage/wasm_storage.rs +++ b/mm2src/mm2_gui_storage/src/account/storage/wasm_storage.rs @@ -11,7 +11,6 @@ use mm2_number::BigDecimal; use serde::{Deserialize, Serialize}; use std::collections::{BTreeMap, BTreeSet}; -const DB_NAME: &str = "gui_account_storage"; const DB_VERSION: u32 = 1; type AccountDbLocked<'a> = DbLocked<'a, AccountDb>; @@ -342,7 +341,7 @@ struct AccountDb { #[async_trait] impl DbInstance for AccountDb { - fn db_name() -> &'static str { DB_NAME } + const DB_NAME: &'static str = "gui_account_storage"; async fn init(db_id: DbIdentifier) -> InitDbResult { let inner = IndexedDbBuilder::new(db_id) diff --git a/mm2src/mm2_main/src/lp_ordermatch/ordermatch_wasm_db.rs b/mm2src/mm2_main/src/lp_ordermatch/ordermatch_wasm_db.rs index 812a802999..c96c2ef024 100644 --- a/mm2src/mm2_main/src/lp_ordermatch/ordermatch_wasm_db.rs +++ b/mm2src/mm2_main/src/lp_ordermatch/ordermatch_wasm_db.rs @@ -9,7 +9,6 @@ pub use mm2_db::indexed_db::{cursor_prelude, DbTransactionError, DbTransactionRe pub use tables::{MyActiveMakerOrdersTable, MyActiveTakerOrdersTable, MyFilteringHistoryOrdersTable, MyHistoryOrdersTable}; -const DB_NAME: &str = "ordermatch"; const DB_VERSION: u32 = 1; pub struct OrdermatchDb { @@ -18,7 +17,7 @@ pub struct OrdermatchDb { #[async_trait] impl DbInstance for OrdermatchDb { - fn db_name() -> &'static str { DB_NAME } + const DB_NAME: &'static str = "ordermatch"; async fn init(db_id: DbIdentifier) -> InitDbResult { let inner = IndexedDbBuilder::new(db_id) diff --git a/mm2src/mm2_main/src/lp_swap/swap_wasm_db.rs b/mm2src/mm2_main/src/lp_swap/swap_wasm_db.rs index d680ba4eb8..4b1b16aaf1 100644 --- a/mm2src/mm2_main/src/lp_swap/swap_wasm_db.rs +++ b/mm2src/mm2_main/src/lp_swap/swap_wasm_db.rs @@ -8,7 +8,6 @@ pub use mm2_db::indexed_db::{cursor_prelude, DbTransactionError, DbTransactionRe ItemId}; pub use tables::{MySwapsFiltersTable, SavedSwapTable, SwapLockTable}; -const DB_NAME: &str = "swap"; const DB_VERSION: u32 = 1; pub struct SwapDb { @@ -17,7 +16,7 @@ pub struct SwapDb { #[async_trait] impl DbInstance for SwapDb { - fn db_name() -> &'static str { DB_NAME } + const DB_NAME: &'static str = "swap"; async fn init(db_id: DbIdentifier) -> InitDbResult { let inner = IndexedDbBuilder::new(db_id) From 34229a9a3a8a4c28ddb31996297038404976c4b8 Mon Sep 17 00:00:00 2001 From: laruh Date: Fri, 13 Oct 2023 15:02:05 +0700 Subject: [PATCH 39/39] dont send scan spam/phishing req in refresh funct if value is `true` --- mm2src/coins/nft.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index c4543a0312..0f0cb942e0 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -427,8 +427,12 @@ pub async fn refresh_nft_metadata(ctx: MmArc, req: RefreshMetadataReq) -> MmResu nft_db.common.last_metadata_sync = moralis_meta.common.last_metadata_sync; nft_db.common.possible_spam = moralis_meta.common.possible_spam; nft_db.uri_meta = uri_meta; - refresh_possible_spam(&storage, &req.chain, &mut nft_db, &req.url_antispam).await?; - refresh_possible_phishing(&storage, &req.chain, domains, &mut nft_db, &req.url_antispam).await?; + if !nft_db.common.possible_spam { + refresh_possible_spam(&storage, &req.chain, &mut nft_db, &req.url_antispam).await?; + }; + if !nft_db.possible_phishing { + refresh_possible_phishing(&storage, &req.chain, domains, &mut nft_db, &req.url_antispam).await?; + }; storage .refresh_nft_metadata(&moralis_meta.chain, nft_db.clone()) .await?; @@ -481,6 +485,9 @@ where let spam_res = send_spam_request(chain, url_antispam, address_hex.clone()).await?; if let Some(true) = spam_res.result.get(&nft_db.common.token_address) { nft_db.common.possible_spam = true; + storage + .update_nft_spam_by_token_address(chain, address_hex.clone(), true) + .await?; storage .update_transfer_spam_by_token_address(chain, address_hex, true) .await?;