Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 18 additions & 5 deletions .github/workflows/main.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
name: Main
name: CI

on:
push:
Expand All @@ -11,11 +11,24 @@ env:

jobs:
build:
runs-on: self-hosted
runs-on: ubuntu-latest

steps:
- uses: actions/checkout@v4
- name: Build
run: cargo build --release

- name: Install stable toolchain
uses: dtolnay/rust-toolchain@stable
with:
components: clippy, rustfmt

- name: Cache cargo artifacts
uses: Swatinem/rust-cache@v2

- name: Check formatting
run: cargo fmt --all -- --check

# - name: Run clippy
# run: cargo clippy --all-targets --all-features -- -D warnings

- name: Run tests
run: cargo test --verbose
run: cargo test --workspace --all-features --verbose
3 changes: 0 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,6 @@ It contains a variety of PBF readers for a variety of scenarios. For example, a
reader with an index can locate and read elements more efficiently. It also provides
a PBF writer that can write PBF data to a file.

Since this crate uses the btree_cursors feature, it requires you to use the **nightly**
version of rust.

- Written in pure Rust
- Provides an indexing feature to the PBF to greatly improve read performance.

Expand Down
8 changes: 4 additions & 4 deletions pbf-craft-cli/src/commands/boundary.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,12 @@ pub struct BoundaryCommand {

impl BoundaryCommand {
pub fn run(self) {
let mut reader =
PbfReader::from_path(&self.file).expect(&format!("No such file: {}", self.file));
let mut reader = PbfReader::from_path(&self.file)
.unwrap_or_else(|_| panic!("No such file: {}", self.file));

let mut polygons: Vec<Polygon> = Vec::new();
while let Some(blob_data) = reader.read_next_blob() {
if blob_data.nodes.len() > 0 {
if !blob_data.nodes.is_empty() {
let points: Vec<geo::Point> = blob_data
.nodes
.into_iter()
Expand All @@ -37,6 +37,6 @@ impl BoundaryCommand {
let geometry: Geometry<f64> = boundary.into();
let geojson = Value::from(&geometry);
dark_yellow_ln!("---------");
println!("{}", geojson.to_string());
println!("{}", geojson);
}
}
7 changes: 2 additions & 5 deletions pbf-craft-cli/src/commands/diff.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
use std::fs::File;

use clap::Args;
use csv;
use serde::{Deserialize, Serialize};

use pbf_craft::models::{Element, ElementType};
Expand Down Expand Up @@ -42,11 +41,9 @@ impl DiffCommand {
csv::WriterBuilder::new().from_writer(File::create(&self.output).unwrap());

let mut source = IterableReader::from_path(&self.source)
.expect(&format!("No such file: {}", self.source))
.into_iter();
.unwrap_or_else(|_| panic!("No such file: {}", self.source));
let mut target = IterableReader::from_path(&self.target)
.expect(&format!("No such file: {}", self.target))
.into_iter();
.unwrap_or_else(|_| panic!("No such file: {}", self.target));

let mut source_element_cnt = source.next();
let mut target_element_cnt = target.next();
Expand Down
15 changes: 8 additions & 7 deletions pbf-craft-cli/src/commands/search.rs
Original file line number Diff line number Diff line change
Expand Up @@ -52,15 +52,13 @@ impl SearchCommand {
}
let element_type = element_type_result.unwrap();

if self.exact.is_none() || self.exact.unwrap() == true {
if self.exact.is_none() || self.exact.unwrap() {
let mut indexed_reader =
IndexedReader::from_path(&self.file).expect("Indexed reader loading failed");
let find_result = indexed_reader.find(&element_type, *elid).unwrap();
match find_result {
Some(ec) => {
let mut list = Vec::new();
list.push(ec);
list
vec![ec]
}
None => Vec::with_capacity(0),
}
Expand All @@ -75,7 +73,8 @@ impl SearchCommand {
return true;
}
}
return false;

false
}
(Element::Way(way), ElementType::Way) => way.id == *elid,
(Element::Relation(relation), ElementType::Relation) => {
Expand All @@ -88,7 +87,8 @@ impl SearchCommand {
return true;
}
}
return false;

false
}
_ => false,
})
Expand Down Expand Up @@ -133,7 +133,8 @@ impl SearchCommand {
return way.way_nodes.iter().any(|ref_node| ref_node.id == first)
&& way.way_nodes.iter().any(|ref_node| ref_node.id == second);
}
return false;

false
})
.expect("node pair error")
} else {
Expand Down
1 change: 0 additions & 1 deletion pbf-craft-cli/src/commands/with_deps.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ use std::str::FromStr;

use clap::Args;
use colored_json::prelude::*;
use serde_json;

use pbf_craft::models::{Element, ElementType};
use pbf_craft::readers::IndexedReader;
Expand Down
113 changes: 57 additions & 56 deletions pbf-craft-cli/src/db/db_reader.rs
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,19 @@ pub enum DbElementType {
Relation,
}

impl Into<ElementType> for DbElementType {
fn into(self) -> ElementType {
match self {
// impl Into<ElementType> for DbElementType {
// fn into(self) -> ElementType {
// match self {
// DbElementType::Node => ElementType::Node,
// DbElementType::Way => ElementType::Way,
// DbElementType::Relation => ElementType::Relation,
// }
// }
// }

impl From<DbElementType> for ElementType {
fn from(value: DbElementType) -> Self {
match value {
DbElementType::Node => ElementType::Node,
DbElementType::Way => ElementType::Way,
DbElementType::Relation => ElementType::Relation,
Expand All @@ -37,7 +47,7 @@ impl DatabaseReader {
let mut config = Config::new();
let _ = config
.host(&host)
.port(port.clone())
.port(port)
.dbname(&dbname)
.user(&user)
.password(&password);
Expand Down Expand Up @@ -83,33 +93,28 @@ impl DatabaseReader {
let mut current_tag_id = 0;
let mut current_tag: Option<Tag> = None;
for node_row in node_cursor {
let mut node = Node::default();
node.id = node_row.get(0);
let latitude: i32 = node_row.get(1);
let longitude: i32 = node_row.get(2);
node.latitude = latitude as i64 * 100;
node.longitude = longitude as i64 * 100;
node.changeset_id = node_row.get(3);
let timestamp: NaiveDateTime = node_row.get(4);
let utc_timestamp: DateTime<Utc> = DateTime::from_naive_utc_and_offset(timestamp, Utc);
node.timestamp = Some(utc_timestamp);
let version: i64 = node_row.get(5);
node.version = version as i32;
node.visible = node_row.get(6);
let user_id: i64 = node_row.get(7);
let user_name: String = node_row.get(8);
node.user = Some(OsmUser {
id: user_id as i32,
name: user_name,
});
let mut node = Node {
id: node_row.get(0),
latitude: node_row.get::<_, i32>(1) as i64 * 100,
longitude: node_row.get::<_, i32>(2) as i64 * 100,
changeset_id: node_row.get(3),
timestamp: Some(DateTime::from_naive_utc_and_offset(node_row.get(4), Utc)),
version: node_row.get::<_, i64>(5) as i32,
visible: node_row.get(6),
user: Some(OsmUser {
id: node_row.get::<_, i64>(7) as i32,
name: node_row.get(8),
}),
..Default::default()
};

if node.id == current_tag_id && current_tag.is_some() {
node.tags.push(current_tag.unwrap());
current_tag = None;
}
while current_tag_id <= node.id || current_tag.is_none() {
let has_tag = tag_iter.next();
if let None = has_tag {
if has_tag.is_none() {
break;
}
let tag_row = has_tag.unwrap();
Expand Down Expand Up @@ -163,29 +168,26 @@ impl DatabaseReader {
let mut current_mem_id = 0;
let mut current_mem: Option<WayNode> = None;
for el_row in el_cursor {
let mut way = Way::default();
way.id = el_row.get(0);
way.changeset_id = el_row.get(1);
let timestamp: NaiveDateTime = el_row.get(2);
let utc_timestamp: DateTime<Utc> = DateTime::from_naive_utc_and_offset(timestamp, Utc);
way.timestamp = Some(utc_timestamp);
let version: i64 = el_row.get(3);
way.version = version as i32;
way.visible = el_row.get(4);
let user_id: i64 = el_row.get(5);
let user_name: String = el_row.get(6);
way.user = Some(OsmUser {
id: user_id as i32,
name: user_name,
});
let mut way = Way {
id: el_row.get(0),
changeset_id: el_row.get(1),
timestamp: Some(DateTime::from_naive_utc_and_offset(el_row.get(2), Utc)),
version: el_row.get::<_, i64>(3) as i32,
visible: el_row.get(4),
user: Some(OsmUser {
id: el_row.get::<_, i64>(5) as i32,
name: el_row.get(6),
}),
..Default::default()
};

if current_tag_id == way.id && current_tag.is_some() {
way.tags.push(current_tag.unwrap());
current_tag = None;
}
while current_tag_id <= way.id || current_tag.is_none() {
let has_tag = tag_iter.next();
if let None = has_tag {
if has_tag.is_none() {
break;
}
let tag_row = has_tag.unwrap();
Expand All @@ -206,7 +208,7 @@ impl DatabaseReader {
}
while current_mem_id <= way.id || current_mem.is_none() {
let has_mem = member_iter.next();
if let None = has_mem {
if has_mem.is_none() {
break;
}
let mem_row = has_mem.unwrap();
Expand Down Expand Up @@ -265,29 +267,28 @@ impl DatabaseReader {
let mut current_mem_id = 0;
let mut current_mem: Option<RelationMember> = None;
for el_row in el_cursor {
let mut relation = Relation::default();
relation.id = el_row.get(0);
relation.changeset_id = el_row.get(1);
let timestamp: NaiveDateTime = el_row.get(2);
let utc_timestamp: DateTime<Utc> = DateTime::from_naive_utc_and_offset(timestamp, Utc);
relation.timestamp = Some(utc_timestamp);
let version: i64 = el_row.get(3);
relation.version = version as i32;
relation.visible = el_row.get(4);
let user_id: i64 = el_row.get(5);
let user_name: String = el_row.get(6);
relation.user = Some(OsmUser {
id: user_id as i32,
name: user_name,
});
let mut relation = Relation {
id: el_row.get(0),
changeset_id: el_row.get(1),
timestamp: Some(utc_timestamp),
version: el_row.get::<_, i64>(3) as i32,
visible: el_row.get(4),
user: Some(OsmUser {
id: el_row.get::<_, i64>(5) as i32,
name: el_row.get(6),
}),
..Default::default()
};

if relation.id == current_tag_id && current_tag.is_some() {
relation.tags.push(current_tag.unwrap());
current_tag = None;
}
while current_tag_id <= relation.id || current_tag.is_none() {
let has_tag = tag_iter.next();
if let None = has_tag {
if has_tag.is_none() {
break;
}
let tag_row = has_tag.unwrap();
Expand All @@ -308,7 +309,7 @@ impl DatabaseReader {
}
while current_mem_id <= relation.id || current_mem.is_none() {
let has_mem = member_iter.next();
if let None = has_mem {
if has_mem.is_none() {
break;
}
let mem_row = has_mem.unwrap();
Expand Down
9 changes: 4 additions & 5 deletions pbf-craft-cli/src/db/paging_cursor.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
use std::mem;
use std::vec::IntoIter;

use postgres::{Client, Portal, Row, Transaction};
Expand Down Expand Up @@ -29,21 +28,21 @@ impl<'client> PagingCursor<'client> {
pub fn new(sql: &str, client: &'client mut Client) -> PagingCursor<'client> {
let mut transaction = client.transaction().unwrap();
let portal = transaction.bind(sql, &[]).unwrap();
let cursor = Self {

Self {
transaction: Some(transaction),
portal,
limit: 32000,
eof: false,
cache: Vec::with_capacity(0).into_iter(),
};
return cursor;
}
}

fn fetch_next(&mut self) -> anyhow::Result<Vec<Row>> {
if let Some(trans) = &mut self.transaction {
let rows = trans.query_portal(&self.portal, self.limit as i32)?;
if rows.len() < self.limit {
let trans = mem::replace(&mut self.transaction, None);
let trans = self.transaction.take();
trans.unwrap().commit()?;
self.eof = true;
}
Expand Down
1 change: 0 additions & 1 deletion pbf-craft-cli/src/main.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
mod commands;
mod db;

use env_logger;
use std::time::Instant;

use clap::Parser;
Expand Down
3 changes: 0 additions & 3 deletions pbf-craft/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,6 @@ It contains a variety of PBF readers for a variety of scenarios. For example, a
reader with an index can locate and read elements more efficiently. It also provides
a PBF writer that can write PBF data to a file.

Since this crate uses the btree_cursors feature, it requires you to use the **nightly**
version of rust.

- Written in pure Rust
- Provides an indexing feature to the PBF to greatly improve read performance.

Expand Down
2 changes: 1 addition & 1 deletion pbf-craft/build.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {

protobuf_codegen_pure::Codegen::new()
.out_dir(&out_dir)
.inputs(&proto_files)
.inputs(proto_files)
.include("src/proto")
.run()?;

Expand Down
Loading
Loading