Skip to content

Commit add7dc5

Browse files
committed
Add place import stats
1 parent ae00d89 commit add7dc5

6 files changed

Lines changed: 362 additions & 5 deletions

File tree

docs/rpc/analytics/README.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ The RPC API provides a [JSON-RPC 2.0](https://www.jsonrpc.org/specification) int
44

55
## Methods
66

7-
- [dashboard](dashboard.md): Get a high-level analytics snapshot (place activity over 1/7/30 day windows).
7+
- [dashboard](dashboard.md): Get a high-level analytics snapshot (place activity, imports by origin, log stats, disk usage, LND balances, and recent sync runs over 1/7/30 day windows).
88
- [get_report](get_report.md): Get analytics report comparing place statistics between two dates.
99
- [get_daily_infra_report](get_daily_infra_report.md): Get daily infrastructure report.
1010
- [get_top_clients](get_top_clients.md): Get top API consumers.

docs/rpc/analytics/dashboard.md

Lines changed: 44 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
## Description
44

5-
Returns a high-level analytics dashboard snapshot, including the time the report took to generate, counts of places added, updated, and deleted over the last 1, 7, and 30 days (from the `element_event` log), log database stats (file size, number of logged requests, the 10 most-called RPC methods, and the 10 most-called REST API endpoints over the last 24 hours), disk usage stats for the host's real block devices, on-chain and Lightning channel balances probed from the LND node, and the 10 most recent OSM sync runs recorded in the `sync` log table.
5+
Returns a high-level analytics dashboard snapshot, including the time the report took to generate, counts of places added, updated, and deleted over the last 1, 7, and 30 days (from the `element_event` log), counts of imported places grouped by import origin over the same windows (from the `place_submission` table), log database stats (file size, number of logged requests, the 10 most-called RPC methods, and the 10 most-called REST API endpoints over the last 24 hours), disk usage stats for the host's real block devices, on-chain and Lightning channel balances probed from the LND node, and the 10 most recent OSM sync runs recorded in the `sync` log table.
66

77
## Params
88

@@ -34,6 +34,44 @@ Returns a high-level analytics dashboard snapshot, including the time the report
3434
"d30": 20
3535
}
3636
},
37+
"imports": [
38+
{
39+
"origin": "square",
40+
"total": {
41+
"d1": 50,
42+
"d7": 300,
43+
"d30": 1200
44+
},
45+
"pending": {
46+
"d1": 10,
47+
"d7": 40,
48+
"d30": 80
49+
},
50+
"revoked": {
51+
"d1": 1,
52+
"d7": 4,
53+
"d30": 15
54+
}
55+
},
56+
{
57+
"origin": "coinos",
58+
"total": {
59+
"d1": 20,
60+
"d7": 120,
61+
"d30": 500
62+
},
63+
"pending": {
64+
"d1": 5,
65+
"d7": 25,
66+
"d30": 60
67+
},
68+
"revoked": {
69+
"d1": 0,
70+
"d7": 2,
71+
"d30": 6
72+
}
73+
}
74+
],
3775
"logs": {
3876
"file_size_bytes": 2230968320,
3977
"requests": {
@@ -125,6 +163,11 @@ Returns a high-level analytics dashboard snapshot, including the time the report
125163
- `places.added.d1` / `d7` / `d30`: Number of `create` events recorded in the last 1, 7, and 30 days
126164
- `places.updated.d1` / `d7` / `d30`: Number of `update` events recorded in the last 1, 7, and 30 days
127165
- `places.deleted.d1` / `d7` / `d30`: Number of `delete` events recorded in the last 1, 7, and 30 days
166+
- `imports`: Imported places grouped by `origin` (e.g. `square`, `coinos`, `btcpayserver`), ordered alphabetically by `origin`. Only origins that had at least one submission created within the largest window (`d30`) are included. Each entry contains:
167+
- `origin`: Name of the import origin (matches `place_submission.origin`)
168+
- `total.d1` / `d7` / `d30`: Number of submissions created in the last 1, 7, and 30 days, regardless of state
169+
- `pending.d1` / `d7` / `d30`: Number of submissions created in the last 1, 7, and 30 days that are still open (`closed_at IS NULL AND revoked = 0`)
170+
- `revoked.d1` / `d7` / `d30`: Number of submissions created in the last 1, 7, and 30 days that have been explicitly revoked (`revoked = 1`)
128171
- `logs.file_size_bytes`: Size of the `log.db` file on disk in bytes (0 if the file is missing)
129172
- `logs.requests.d1` / `d7` / `d30`: Number of HTTP requests logged in the last 1, 7, and 30 days
130173
- `logs.top_rpcs`: Up to 10 most-called RPC methods on the `/rpc` endpoint over the last 24 hours, ordered by `count` descending (most-called first). Each entry contains:

src/db/main/place_submission/blocking_queries.rs

Lines changed: 123 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
use super::schema::{self, Columns, PlaceSubmission};
1+
use super::schema::{self, Columns, OriginSubmissionCounts, PlaceSubmission};
22
use crate::Result;
33
use geojson::JsonObject;
44
use rusqlite::{named_params, params, Connection, OptionalExtension};
@@ -70,6 +70,44 @@ pub fn select_open_and_not_revoked(conn: &Connection) -> Result<Vec<PlaceSubmiss
7070
.map_err(Into::into)
7171
}
7272

73+
pub fn select_origin_counts_since(
74+
since: OffsetDateTime,
75+
conn: &Connection,
76+
) -> Result<Vec<OriginSubmissionCounts>> {
77+
let sql = format!(
78+
r#"
79+
SELECT
80+
{origin} AS origin,
81+
COUNT(*) AS total,
82+
SUM(CASE WHEN {closed_at} IS NULL AND {revoked} = 0 THEN 1 ELSE 0 END) AS pending,
83+
SUM(CASE WHEN {revoked} = 1 THEN 1 ELSE 0 END) AS revoked
84+
FROM {table}
85+
WHERE {created_at} >= ?1
86+
GROUP BY {origin}
87+
ORDER BY {origin}
88+
"#,
89+
table = schema::TABLE_NAME,
90+
origin = Columns::Origin.as_str(),
91+
closed_at = Columns::ClosedAt.as_str(),
92+
revoked = Columns::Revoked.as_str(),
93+
created_at = Columns::CreatedAt.as_str(),
94+
);
95+
let mut stmt = conn.prepare(&sql)?;
96+
let rows = stmt.query_map(params![since.format(&Rfc3339)?], |row| {
97+
Ok(OriginSubmissionCounts {
98+
origin: row.get("origin")?,
99+
total: row.get("total")?,
100+
pending: row.get("pending")?,
101+
revoked: row.get("revoked")?,
102+
})
103+
})?;
104+
let mut res = vec![];
105+
for row in rows {
106+
res.push(row?);
107+
}
108+
Ok(res)
109+
}
110+
73111
pub fn select_by_id(id: i64, conn: &Connection) -> Result<PlaceSubmission> {
74112
let sql = format!(
75113
r#"
@@ -248,6 +286,7 @@ mod test {
248286
use crate::Result;
249287
use geojson::JsonObject;
250288
use serde_json::Map;
289+
use time::macros::datetime;
251290
use time::OffsetDateTime;
252291

253292
#[test]
@@ -433,4 +472,87 @@ mod test {
433472
);
434473
Ok(())
435474
}
475+
476+
#[test]
477+
fn select_origin_counts_since_groups_by_origin() -> Result<()> {
478+
let conn = conn();
479+
480+
let insert = |origin: &str, external_id: &str| -> Result<i64> {
481+
let args = InsertArgs {
482+
origin: origin.to_string(),
483+
external_id: external_id.to_string(),
484+
lat: 1.0,
485+
lon: 2.0,
486+
category: "cafe".to_string(),
487+
name: "Place".to_string(),
488+
extra_fields: Map::new(),
489+
};
490+
Ok(super::insert(&args, &conn)?.id)
491+
};
492+
493+
let square_recent_1 = insert("square", "1")?;
494+
let square_recent_2 = insert("square", "2")?;
495+
let square_old = insert("square", "3")?;
496+
let coinos_recent = insert("coinos", "1")?;
497+
let coinos_closed = insert("coinos", "2")?;
498+
let coinos_revoked = insert("coinos", "3")?;
499+
let coinos_old = insert("coinos", "4")?;
500+
501+
for id in [square_old, coinos_old] {
502+
conn.execute(
503+
"UPDATE place_submission SET created_at = '2020-01-01T00:00:00Z' WHERE id = ?1",
504+
rusqlite::params![id],
505+
)?;
506+
}
507+
508+
super::set_closed_at(coinos_closed, Some(datetime!(2024-06-01 00:00 UTC)), &conn)?;
509+
super::set_revoked(coinos_revoked, true, &conn)?;
510+
511+
let _ = square_recent_1;
512+
let _ = square_recent_2;
513+
let _ = coinos_recent;
514+
515+
let counts = super::select_origin_counts_since(datetime!(2024-01-01 00:00 UTC), &conn)?;
516+
assert_eq!(
517+
vec![
518+
super::schema::OriginSubmissionCounts {
519+
origin: "coinos".to_string(),
520+
total: 3,
521+
pending: 1,
522+
revoked: 1,
523+
},
524+
super::schema::OriginSubmissionCounts {
525+
origin: "square".to_string(),
526+
total: 2,
527+
pending: 2,
528+
revoked: 0,
529+
},
530+
],
531+
counts
532+
);
533+
534+
let counts = super::select_origin_counts_since(datetime!(2019-01-01 00:00 UTC), &conn)?;
535+
assert_eq!(
536+
vec![
537+
super::schema::OriginSubmissionCounts {
538+
origin: "coinos".to_string(),
539+
total: 4,
540+
pending: 2,
541+
revoked: 1,
542+
},
543+
super::schema::OriginSubmissionCounts {
544+
origin: "square".to_string(),
545+
total: 3,
546+
pending: 3,
547+
revoked: 0,
548+
},
549+
],
550+
counts
551+
);
552+
553+
let counts = super::select_origin_counts_since(datetime!(2030-01-01 00:00 UTC), &conn)?;
554+
assert!(counts.is_empty());
555+
556+
Ok(())
557+
}
436558
}

src/db/main/place_submission/queries.rs

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
use crate::{
22
db::main::place_submission::{
3-
blocking_queries, blocking_queries::InsertArgs, schema::PlaceSubmission,
3+
blocking_queries,
4+
blocking_queries::InsertArgs,
5+
schema::{OriginSubmissionCounts, PlaceSubmission},
46
},
57
Result,
68
};
@@ -22,6 +24,16 @@ pub async fn select_open_and_not_revoked(pool: &Pool) -> Result<Vec<PlaceSubmiss
2224
.await?
2325
}
2426

27+
pub async fn select_origin_counts_since(
28+
since: OffsetDateTime,
29+
pool: &Pool,
30+
) -> Result<Vec<OriginSubmissionCounts>> {
31+
pool.get()
32+
.await?
33+
.interact(move |conn| blocking_queries::select_origin_counts_since(since, conn))
34+
.await?
35+
}
36+
2537
pub async fn select_by_id(id: i64, pool: &Pool) -> Result<PlaceSubmission> {
2638
pool.get()
2739
.await?

src/db/main/place_submission/schema.rs

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,14 @@ pub struct PlaceSubmission {
6161
pub deleted_at: Option<OffsetDateTime>,
6262
}
6363

64+
#[derive(Debug, PartialEq)]
65+
pub struct OriginSubmissionCounts {
66+
pub origin: String,
67+
pub total: i64,
68+
pub pending: i64,
69+
pub revoked: i64,
70+
}
71+
6472
impl PlaceSubmission {
6573
pub fn projection() -> &'static str {
6674
static PROJECTION: OnceLock<String> = OnceLock::new();

0 commit comments

Comments
 (0)