feat: improve onboarding

Author: Ludo Galabru

components/hord-cli/src/cli/mod.rs

@@ -13,7 +13,7 @@ use crate::db::{
     find_all_transfers_in_block, find_inscription_with_id, find_last_block_inserted,
     find_latest_inscription_block_height, find_lazy_block_at_block_height,
     open_readonly_hord_db_conn, open_readonly_hord_db_conn_rocks_db, open_readwrite_hord_db_conn,
-    open_readwrite_hord_db_conn_rocks_db, initialize_hord_db,
+    open_readwrite_hord_db_conn_rocks_db, initialize_hord_db, get_default_hord_db_file_path,
 };
 use chainhook_sdk::bitcoincore_rpc::{Auth, Client, RpcApi};
 use chainhook_sdk::chainhooks::types::HttpHook;
@@ -261,6 +261,9 @@ struct StartCommand {
 
 #[derive(Subcommand, PartialEq, Clone, Debug)]
 enum HordDbCommand {
+    /// Initialize a new hord db
+    #[clap(name = "new", bin_name = "new")]
+    New(SyncHordDbCommand),    
     /// Catch-up hord db
     #[clap(name = "sync", bin_name = "sync")]
     Sync(SyncHordDbCommand),
@@ -476,21 +479,25 @@ async fn handle_command(opts: Opts, ctx: &Context) -> Result<(), String> {
                     .await?;
             } else {
                 let _ = download_ordinals_dataset_if_required(&config, ctx).await;
+                let mut total_inscriptions = 0;
+                let mut total_transfers = 0;
 
                 let inscriptions_db_conn =
-                    open_readonly_hord_db_conn(&config.expected_cache_path(), &ctx)?;
+                    initialize_hord_db(&config.expected_cache_path(), &ctx);
                 while let Some(block_height) = block_range.pop_front() {
-                    let mut total_transfers = 0;
                     let inscriptions =
                         find_all_inscriptions_in_block(&block_height, &inscriptions_db_conn, &ctx);
                     let mut locations =
                         find_all_transfers_in_block(&block_height, &inscriptions_db_conn, &ctx);
+
+                    let mut total_transfers_in_block = 0;
+
                     for (_, inscription) in inscriptions.iter() {
                         println!("Inscription {} revealed at block #{} (inscription_number {}, ordinal_number {})", inscription.get_inscription_id(), block_height, inscription.inscription_number, inscription.ordinal_number);
                         if let Some(transfers) = locations.remove(&inscription.get_inscription_id())
                         {
                             for t in transfers.iter().skip(1) {
-                                total_transfers += 1;
+                                total_transfers_in_block += 1;
                                 println!(
                                     "\t→ Transferred in transaction {}",
                                     t.transaction_identifier_location.hash
@@ -501,21 +508,28 @@ async fn handle_command(opts: Opts, ctx: &Context) -> Result<(), String> {
                     for (inscription_id, transfers) in locations.iter() {
                         println!("Inscription {}", inscription_id);
                         for t in transfers.iter() {
-                            total_transfers += 1;
+                            total_transfers_in_block += 1;
                             println!(
                                 "\t→ Transferred in transaction {}",
                                 t.transaction_identifier_location.hash
                             );
                         }
                     }
-                    if total_transfers > 0 && inscriptions.len() > 0 {
+                    if total_transfers_in_block > 0 && inscriptions.len() > 0 {
                         println!(
-                            "Inscriptions revealed: {}, inscriptions transferred: {total_transfers}",
+                            "Inscriptions revealed: {}, inscriptions transferred: {total_transfers_in_block}",
                             inscriptions.len()
                         );
                         println!("-----");
                     }
+
+                    total_inscriptions += inscriptions.len();
+                    total_transfers += total_transfers_in_block;
                 }
+                if total_transfers == 0 && total_inscriptions == 0 {
+                    let db_file_path = get_default_hord_db_file_path(&config.expected_cache_path());
+                    warn!(ctx.expect_logger(), "No data available. Check the validity of the range being scanned and the validity of your local database {}", db_file_path.display());
+                }    
             }
         }
         Command::Scan(ScanCommand::Inscription(cmd)) => {
@@ -620,6 +634,10 @@ async fn handle_command(opts: Opts, ctx: &Context) -> Result<(), String> {
                 println!("Created file Hord.toml");
             }
         },
+        Command::Db(HordDbCommand::New(cmd)) => {
+            let config = Config::default(false, false, false, &cmd.config_path)?;
+            initialize_hord_db(&config.expected_cache_path(), &ctx);
+        },
         Command::Db(HordDbCommand::Sync(_cmd)) => unimplemented!(),
         Command::Db(HordDbCommand::Repair(subcmd)) => match subcmd {
             RepairCommand::Blocks(cmd) => {

components/hord-cli/src/config/file.rs

@@ -2,10 +2,10 @@
 pub struct ConfigFile {
     pub storage: StorageConfigFile,
     pub http_api: Option<PredicatesApiConfigFile>,
-    pub event_source: Option<Vec<EventSourceConfigFile>>,
     pub limits: LimitsConfigFile,
     pub network: NetworkConfigFile,
     pub logs: Option<LogConfigFile>,
+    pub bootstrap: Option<BootstrapConfigFile>,
 }
 
 #[derive(Deserialize, Debug, Clone)]
@@ -28,13 +28,8 @@ pub struct PredicatesApiConfigFile {
 }
 
 #[derive(Deserialize, Debug, Clone)]
-pub struct EventSourceConfigFile {
-    pub source_type: Option<String>,
-    pub stacks_node_url: Option<String>,
-    pub chainhook_node_url: Option<String>,
-    pub polling_delay: Option<u32>,
-    pub tsv_file_path: Option<String>,
-    pub tsv_file_url: Option<String>,
+pub struct BootstrapConfigFile {
+    pub download_url: Option<String>,
 }
 
 #[derive(Deserialize, Debug, Clone)]

components/hord-cli/src/config/generator.rs

@@ -34,6 +34,11 @@ max_number_of_processing_threads = 16
 bitcoin_concurrent_http_requests_max = 16
 max_caching_memory_size_mb = 32000
 
+# Disable the following section if the state
+# must be built locally
+[bootstrap]
+download_url = "https://archive.hiro.so/mainnet/chainhooks/hord.sqlite"
+
 [logs]
 ordinals_internals = true
 chainhook_internals = true

components/hord-cli/src/config/mod.rs

@@ -27,9 +27,9 @@ pub const BITCOIN_MAX_PREDICATE_REGISTRATION: usize = 50;
 pub struct Config {
     pub storage: StorageConfig,
     pub http_api: PredicatesApi,
-    pub event_sources: Vec<EventSourceConfig>,
     pub limits: LimitsConfig,
     pub network: IndexerConfig,
+    pub bootstrap: BootstrapConfig,
     pub logs: LogConfig,
 }
 
@@ -58,9 +58,9 @@ pub struct PredicatesApiConfig {
 }
 
 #[derive(Clone, Debug)]
-pub enum EventSourceConfig {
-    OrdinalsSqlitePath(PathConfig),
-    OrdinalsSqliteUrl(UrlConfig),
+pub enum BootstrapConfig {
+    Build,
+    Download(String),
 }
 
 #[derive(Clone, Debug)]
@@ -152,21 +152,13 @@ impl Config {
             _ => return Err("network.mode not supported".to_string()),
         };
 
-        let mut event_sources = vec![];
-        for source in config_file.event_source.unwrap_or(vec![]).iter_mut() {
-            if let Some(dst) = source.tsv_file_path.take() {
-                let mut file_path = PathBuf::new();
-                file_path.push(dst);
-                event_sources.push(EventSourceConfig::OrdinalsSqlitePath(PathConfig {
-                    file_path,
-                }));
-                continue;
+        let bootstrap = match config_file.bootstrap {
+            Some(bootstrap) => match bootstrap.download_url {
+                Some(ref url) => BootstrapConfig::Download(url.to_string()),
+                None => BootstrapConfig::Build
             }
-            if let Some(file_url) = source.tsv_file_url.take() {
-                event_sources.push(EventSourceConfig::OrdinalsSqliteUrl(UrlConfig { file_url }));
-                continue;
-            }
-        }
+            None => BootstrapConfig::Build
+        };
 
         let config = Config {
             storage: StorageConfig {
@@ -185,7 +177,7 @@ impl Config {
                     }),
                 },
             },
-            event_sources,
+            bootstrap,
             limits: LimitsConfig {
                 max_number_of_stacks_predicates: config_file
                     .limits
@@ -248,28 +240,11 @@ impl Config {
         Ok(config)
     }
 
-    pub fn is_initial_ingestion_required(&self) -> bool {
-        for source in self.event_sources.iter() {
-            match source {
-                EventSourceConfig::OrdinalsSqlitePath(_)
-                | EventSourceConfig::OrdinalsSqliteUrl(_) => return true,
-            }
+    pub fn should_bootstrap_through_download(&self) -> bool {
+        match &self.bootstrap {
+            BootstrapConfig::Build => false,
+            BootstrapConfig::Download(_) => true
         }
-        return false;
-    }
-
-    pub fn add_ordinals_sqlite_remote_source_url(&mut self, file_url: &str) {
-        self.event_sources
-            .push(EventSourceConfig::OrdinalsSqliteUrl(UrlConfig {
-                file_url: file_url.to_string(),
-            }));
-    }
-
-    pub fn add_local_ordinals_sqlite_source(&mut self, file_path: &PathBuf) {
-        self.event_sources
-            .push(EventSourceConfig::OrdinalsSqlitePath(PathConfig {
-                file_path: file_path.clone(),
-            }));
     }
 
     pub fn expected_api_database_uri(&self) -> &str {
@@ -289,13 +264,11 @@ impl Config {
         destination_path
     }
 
-    fn expected_remote_ordinals_sqlite_base_url(&self) -> &String {
-        for source in self.event_sources.iter() {
-            if let EventSourceConfig::OrdinalsSqliteUrl(config) = source {
-                return &config.file_url;
-            }
+    fn expected_remote_ordinals_sqlite_base_url(&self) -> &str {
+        match &self.bootstrap {
+            BootstrapConfig::Build => unreachable!(),
+            BootstrapConfig::Download(url) => &url
         }
-        panic!("expected remote-tsv source")
     }
 
     pub fn expected_remote_ordinals_sqlite_sha256(&self) -> String {
@@ -306,29 +279,6 @@ impl Config {
         format!("{}.gz", self.expected_remote_ordinals_sqlite_base_url())
     }
 
-    pub fn rely_on_remote_ordinals_sqlite(&self) -> bool {
-        for source in self.event_sources.iter() {
-            if let EventSourceConfig::OrdinalsSqliteUrl(_config) = source {
-                return true;
-            }
-        }
-        false
-    }
-
-    pub fn should_download_remote_ordinals_sqlite(&self) -> bool {
-        let mut rely_on_remote_tsv = false;
-        let mut remote_tsv_present_locally = false;
-        for source in self.event_sources.iter() {
-            if let EventSourceConfig::OrdinalsSqliteUrl(_config) = source {
-                rely_on_remote_tsv = true;
-            }
-            if let EventSourceConfig::OrdinalsSqlitePath(_config) = source {
-                remote_tsv_present_locally = true;
-            }
-        }
-        rely_on_remote_tsv == true && remote_tsv_present_locally == false
-    }
-
     pub fn default(
         devnet: bool,
         testnet: bool,
@@ -351,7 +301,7 @@ impl Config {
                 working_dir: default_cache_path(),
             },
             http_api: PredicatesApi::Off,
-            event_sources: vec![],
+            bootstrap: BootstrapConfig::Build,
             limits: LimitsConfig {
                 max_number_of_bitcoin_predicates: BITCOIN_MAX_PREDICATE_REGISTRATION,
                 max_number_of_concurrent_bitcoin_scans: BITCOIN_SCAN_THREAD_POOL_SIZE,
@@ -384,7 +334,7 @@ impl Config {
                 working_dir: default_cache_path(),
             },
             http_api: PredicatesApi::Off,
-            event_sources: vec![],
+            bootstrap: BootstrapConfig::Build,
             limits: LimitsConfig {
                 max_number_of_bitcoin_predicates: BITCOIN_MAX_PREDICATE_REGISTRATION,
                 max_number_of_concurrent_bitcoin_scans: BITCOIN_SCAN_THREAD_POOL_SIZE,
@@ -417,9 +367,7 @@ impl Config {
                 working_dir: default_cache_path(),
             },
             http_api: PredicatesApi::Off,
-            event_sources: vec![EventSourceConfig::OrdinalsSqliteUrl(UrlConfig {
-                file_url: DEFAULT_MAINNET_ORDINALS_SQLITE_ARCHIVE.into(),
-            })],
+            bootstrap: BootstrapConfig::Download(DEFAULT_MAINNET_ORDINALS_SQLITE_ARCHIVE.to_string()),
             limits: LimitsConfig {
                 max_number_of_bitcoin_predicates: BITCOIN_MAX_PREDICATE_REGISTRATION,
                 max_number_of_concurrent_bitcoin_scans: BITCOIN_SCAN_THREAD_POOL_SIZE,

components/hord-cli/src/db/mod.rs

@@ -23,7 +23,7 @@ use crate::{
     core::protocol::inscription_parsing::get_inscriptions_revealed_in_block, ord::sat::Sat,
 };
 
-fn get_default_hord_db_file_path(base_dir: &PathBuf) -> PathBuf {
+pub fn get_default_hord_db_file_path(base_dir: &PathBuf) -> PathBuf {
     let mut destination_path = base_dir.clone();
     destination_path.push("hord.sqlite");
     destination_path
@@ -124,7 +124,7 @@ pub fn initialize_hord_db(path: &PathBuf, ctx: &Context) -> Connection {
     conn
 }
 
-fn create_or_open_readwrite_db(cache_path: &PathBuf, ctx: &Context) -> Connection {
+pub fn create_or_open_readwrite_db(cache_path: &PathBuf, ctx: &Context) -> Connection {
     let path = get_default_hord_db_file_path(&cache_path);
     let open_flags = match std::fs::metadata(&path) {
         Err(e) => {