diff --git a/.gitignore b/.gitignore index 4bd886a..0830f00 100644 --- a/.gitignore +++ b/.gitignore @@ -14,4 +14,5 @@ sherpa-onnx-zipformer-* *.txt sherpa-onnx-v* jniLibs -sys/src/bindings.rs \ No newline at end of file +sys/src/bindings.rs +!sys/dist.txt \ No newline at end of file diff --git a/BUILDING.md b/BUILDING.md index 74632df..9ac20bf 100644 --- a/BUILDING.md +++ b/BUILDING.md @@ -98,6 +98,8 @@ git pull origin master ### Gotachas +On Linux you should set `RUSTFLAGS="-C relocation-model=dynamic-no-pic"` +
When using GPU such as DirectML or Cuda @@ -193,3 +195,15 @@ For debug the build process of sherpa-onnx, please set `BUILD_DEBUG=1` environme ```console gh release create v0.4.1 --title v0.4.1 --generate-notes ``` + +## Calculate sha256 for dist table + +```console +shasum -a 256 | tr 'a-z' 'A-Z' +``` + +## See debug log from build + +``` +BUILD_DEBUG=1 cargo build -vv +``` diff --git a/Cargo.lock b/Cargo.lock index 4f99e65..b5c9423 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "adler2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" + [[package]] name = "aho-corasick" version = "1.1.3" @@ -47,7 +53,7 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ad186efb764318d35165f1758e7dcef3b10628e26d41a44bc5550652e6804391" dependencies = [ - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -57,9 +63,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61a38449feb7068f52bb06c12759005cf459ee52bb4adc1d5a7c4322d716fb19" dependencies = [ "anstyle", - "windows-sys", + "windows-sys 0.52.0", ] +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + [[package]] name = "bindgen" version = "0.69.4" @@ -89,6 +101,42 @@ version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bzip2" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" +dependencies = [ + "bzip2-sys", + "libc", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.11+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "cc" version = "1.1.0" @@ -176,6 +224,44 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422" +[[package]] +name = "cpufeatures" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "608697df725056feaccfa42cffdaeeec3fccc4ffc38358ecd19b243e716a78e0" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + [[package]] name = "either" version = "1.13.0" @@ -189,7 +275,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -202,6 +288,58 @@ dependencies = [ "once_cell", ] +[[package]] +name = "filetime" +version = "0.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35c0522e981e68cbfa8c3f978441a5f34b30b96e146b33cd3359176b50fe8586" +dependencies = [ + "cfg-if", + "libc", + "libredox", + "windows-sys 0.59.0", +] + +[[package]] +name = "flate2" +version = "1.0.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1b589b4dc103969ad3cf85c950899926ec64300a1a46d76c03a6072957036f0" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "form_urlencoded" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + [[package]] name = "glob" version = "0.3.1" @@ -220,7 +358,7 @@ version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" dependencies = [ - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -229,6 +367,16 @@ version = "3.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "62adaabb884c94955b19907d60019f4e145d091c75345379e70d1ee696f7854f" +[[package]] +name = "idna" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + [[package]] name = "indenter" version = "0.3.3" @@ -278,6 +426,17 @@ dependencies = [ "windows-targets", ] +[[package]] +name = "libredox" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" +dependencies = [ + "bitflags", + "libc", + "redox_syscall", +] + [[package]] name = "linux-raw-sys" version = "0.4.14" @@ -302,6 +461,15 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" +[[package]] +name = "miniz_oxide" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" +dependencies = [ + "adler2", +] + [[package]] name = "nom" version = "7.1.3" @@ -318,6 +486,18 @@ version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +[[package]] +name = "percent-encoding" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" + +[[package]] +name = "pkg-config" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" + [[package]] name = "prettyplease" version = "0.2.20" @@ -346,6 +526,15 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "redox_syscall" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b6dfecf2c74bce2466cabf93f6664d6998a69eb21e39f4207930065b27b771f" +dependencies = [ + "bitflags", +] + [[package]] name = "regex" version = "1.10.5" @@ -375,6 +564,21 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" +[[package]] +name = "ring" +version = "0.17.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" +dependencies = [ + "cc", + "cfg-if", + "getrandom", + "libc", + "spin", + "untrusted", + "windows-sys 0.52.0", +] + [[package]] name = "rustc-hash" version = "1.1.0" @@ -391,7 +595,50 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustls" +version = "0.23.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fbb44d7acc4e873d613422379f69f237a1b141928c02f6bc6ccfddddc2d7993" +dependencies = [ + "log", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16f1201b3c9a7ee8039bcadc17b7e605e2945b27eee7631788c1bd2b0643674b" + +[[package]] +name = "rustls-webpki" +version = "0.102.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "sha2" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", ] [[package]] @@ -410,8 +657,13 @@ name = "sherpa-rs-sys" version = "0.4.1" dependencies = [ "bindgen", + "bzip2", "cmake", + "flate2", "glob", + "sha2", + "tar", + "ureq", ] [[package]] @@ -420,12 +672,35 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "socks" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0c3dbbd9ae980613c6dd8e28a9407b50509d3803b57624d5dfe8315218cd58b" +dependencies = [ + "byteorder", + "libc", + "winapi", +] + +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" + [[package]] name = "strsim" version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + [[package]] name = "syn" version = "2.0.70" @@ -437,18 +712,119 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "tar" +version = "0.4.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ff6c40d3aedb5e06b57c6f669ad17ab063dd1e63d977c6a88e7f4dfa4f04020" +dependencies = [ + "filetime", + "libc", + "xattr", +] + +[[package]] +name = "tinyvec" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "445e881f4f6d382d5f27c034e25eb92edd7c784ceab92a0937db7f2e9471b938" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "typenum" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" + +[[package]] +name = "unicode-bidi" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ab17db44d7388991a428b2ee655ce0c212e862eff1768a455c58f9aad6e7893" + [[package]] name = "unicode-ident" version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +[[package]] +name = "unicode-normalization" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "ureq" +version = "2.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b74fc6b57825be3373f7054754755f03ac3a8f5d70015ccad699ba2029956f4a" +dependencies = [ + "base64", + "log", + "once_cell", + "rustls", + "rustls-pki-types", + "socks", + "url", + "webpki-roots", +] + +[[package]] +name = "url" +version = "2.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + [[package]] name = "utf8parse" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "webpki-roots" +version = "0.26.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841c67bff177718f1d4dfefde8d8f0e78f9b6589319ba88312f567fc5841a958" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "which" version = "4.4.2" @@ -461,6 +837,28 @@ dependencies = [ "rustix", ] +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows-sys" version = "0.52.0" @@ -470,6 +868,15 @@ dependencies = [ "windows-targets", ] +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-targets" version = "0.52.6" @@ -533,3 +940,20 @@ name = "windows_x86_64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "xattr" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8da84f1a25939b27f6820d92aed108f83ff920fdf11a7b19366c27c4cda81d4f" +dependencies = [ + "libc", + "linux-raw-sys", + "rustix", +] + +[[package]] +name = "zeroize" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" diff --git a/Cargo.toml b/Cargo.toml index b026277..0621195 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,7 +28,8 @@ sherpa-rs-sys = { path = "sys", version = "0.4.1" } clap = { version = "4.5.8", features = ["derive"] } [features] -default = [] +default = ["download-binaries"] +download-binaries = ["sherpa-rs-sys/download-binaries"] tts = ["sherpa-rs-sys/tts"] cuda = ["sherpa-rs-sys/cuda"] directml = ["sherpa-rs-sys/directml"] diff --git a/README.md b/README.md index e182a5e..0024e35 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,7 @@ Please see [BUILDING.md](BUILDING.md). - `cuda`: enable CUDA support - `directml`: enable DirectML support - `tts`: enable TTS +- `download-binaries`: use prebuilt sherpa-onnx libraries for faster builds. cached. ## Docs diff --git a/sys/Cargo.toml b/sys/Cargo.toml index f537bea..208cb4c 100644 --- a/sys/Cargo.toml +++ b/sys/Cargo.toml @@ -30,14 +30,25 @@ include = [ "src/*.rs", "build.rs", "wrapper.h", + "dist.txt", ] [build-dependencies] bindgen = "0.69.4" cmake = "0.1" glob = "0.3.1" +# Download binaries +ureq = { version = "2.1", optional = true, default-features = false, features = [ + "tls", + "socks-proxy", +] } +tar = { version = "0.4", optional = true } +bzip2 = { version = "0.4.4", optional = true } +flate2 = { version = "1.0", optional = true } +sha2 = { version = "0.10", optional = true } [features] +download-binaries = ["ureq", "tar", "bzip2", "flate2", "sha2"] default = [] tts = [] cuda = [] diff --git a/sys/build.rs b/sys/build.rs index 765ab6f..9803d35 100644 --- a/sys/build.rs +++ b/sys/build.rs @@ -1,9 +1,16 @@ use cmake::Config; use glob::glob; use std::env; +use std::fs; use std::path::{Path, PathBuf}; use std::process::Command; +#[path = "src/internal/dirs.rs"] +mod dirs; +use self::dirs::cache_dir; + +const DIST_TABLE: &str = include_str!("dist.txt"); + macro_rules! debug_log { ($($arg:tt)*) => { if std::env::var("BUILD_DEBUG").is_ok() { @@ -12,6 +19,91 @@ macro_rules! debug_log { }; } +#[cfg(feature = "download-binaries")] +fn fetch_file(source_url: &str) -> Vec { + let resp = ureq::AgentBuilder::new() + .try_proxy_from_env(true) + .build() + .get(source_url) + .timeout(std::time::Duration::from_secs(1800)) + .call() + .unwrap_or_else(|err| panic!("Failed to GET `{source_url}`: {err}")); + + let len = resp + .header("Content-Length") + .and_then(|s| s.parse::().ok()) + .expect("Content-Length header should be present on archive response"); + debug_log!("Fetch file {} {}", source_url, len); + let mut reader = resp.into_reader(); + let mut buffer = Vec::new(); + reader + .read_to_end(&mut buffer) + .unwrap_or_else(|err| panic!("Failed to download from `{source_url}`: {err}")); + assert_eq!(buffer.len(), len); + buffer +} + +#[derive(Debug)] +struct Dist { + url: String, + sha256: String, + name: String, +} + +fn find_dist(target: &str, feature_set: &str) -> Option { + DIST_TABLE + .split('\n') + .skip(1) // Skip headers + .filter(|l| !l.is_empty() && !l.starts_with('#')) + .map(|l| l.split_whitespace().collect::>()) + .find(|row| row[0] == feature_set && row[1] == target) + .map(|row| Dist { + url: row[2].into(), + name: row[3].into(), + sha256: row[4].into(), + }) +} + +#[cfg(feature = "download-binaries")] +fn hex_str_to_bytes(c: impl AsRef<[u8]>) -> Vec { + fn nibble(c: u8) -> u8 { + match c { + b'A'..=b'F' => c - b'A' + 10, + b'a'..=b'f' => c - b'a' + 10, + b'0'..=b'9' => c - b'0', + _ => panic!(), + } + } + + c.as_ref() + .chunks(2) + .map(|n| nibble(n[0]) << 4 | nibble(n[1])) + .collect() +} + +#[cfg(feature = "download-binaries")] +fn verify_file(buf: &[u8], hash: impl AsRef<[u8]>) -> bool { + ::digest(buf)[..] == hex_str_to_bytes(hash) +} + +#[cfg(feature = "download-binaries")] +#[allow(unused)] +fn extract_tgz(buf: &[u8], output: &Path) { + let buf: std::io::BufReader<&[u8]> = std::io::BufReader::new(buf); + let tar = flate2::read::GzDecoder::new(buf); + let mut archive = tar::Archive::new(tar); + archive.unpack(output).expect("Failed to extract .tgz file"); +} + +#[cfg(feature = "download-binaries")] +fn extract_tbz(buf: &[u8], output: &Path) { + debug_log!("extracging tbz to {}", output.display()); + let buf: std::io::BufReader<&[u8]> = std::io::BufReader::new(buf); + let tar = bzip2::read::BzDecoder::new(buf); // Use BzDecoder for .bz2 + let mut archive = tar::Archive::new(tar); + archive.unpack(output).expect("Failed to extract .tbz file"); +} + fn hard_link(src: PathBuf, dst: PathBuf) { std::fs::hard_link(src, dst).unwrap(); } @@ -153,6 +245,7 @@ fn rerun_on_env_changes(vars: &[&str]) { fn main() { let target = env::var("TARGET").unwrap(); + debug_log!("TARGET: {:?}", target); let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap()); let target_dir = get_cargo_target_dir().unwrap(); @@ -276,9 +369,42 @@ fn main() { let sherpa_libs: Vec; let sherpa_libs_kind = if build_shared_libs { "dylib" } else { "static" }; + #[cfg(feature = "download-binaries")] + { + // Try download sherpa libs and set SHERPA_LIB_PATH + if let Some(dist) = find_dist(&target, "none") { + debug_log!("Dist: {:?}", dist); + let mut cache_dir = cache_dir() + .expect("could not determine cache directory") + .join("sherpa-bin") + .join(&target) + .join(&dist.sha256); + if fs::create_dir_all(&cache_dir).is_err() { + cache_dir = env::var("OUT_DIR").unwrap().into(); + } + debug_log!("Cache dir: {}", cache_dir.display()); + let lib_dir = cache_dir.join(&dist.name); + if !lib_dir.exists() { + let downloaded_file = fetch_file(&dist.url); + assert!( + verify_file(&downloaded_file, &dist.sha256), + "hash of downloaded Sherpa-ONNX Runtime binary does not match!" + ); + extract_tbz(&downloaded_file, &cache_dir); + env::set_var("SHERPA_LIB_PATH", cache_dir.join(&dist.name)); + } else { + debug_log!("Skip fetch file. Using cache from {}", lib_dir.display()); + env::set_var("SHERPA_LIB_PATH", cache_dir.join(&dist.name)); + } + } else { + debug_log!("Failed to download binaries") + } + } + if let Ok(sherpa_lib_path) = env::var("SHERPA_LIB_PATH") { // Skip build if SHERPA_LIB_PATH specified debug_log!("Skpping build with Cmake..."); + debug_log!("SHERPA_LIB_PATH: {}", sherpa_lib_path); println!( "cargo:rustc-link-search={}", Path::new(&sherpa_lib_path).join("lib").display() diff --git a/sys/dist.txt b/sys/dist.txt new file mode 100644 index 0000000..be7f186 --- /dev/null +++ b/sys/dist.txt @@ -0,0 +1,6 @@ +# Dist table for prebuilt sherpa-onnx binaries +FEATURES ARCH URL NAME SHA256 SIZE(MB) +none x86_64-pc-windows-msvc https://github.com/k2-fsa/sherpa-onnx/releases/download/v1.10.28/sherpa-onnx-v1.10.28-win-x64-static.tar.bz2 sherpa-onnx-v1.10.28-win-x64-static BECD2A21264C039AA59B7B71D6CD81F33FE732E73251C14391AF5F7E3AE604EA 128 +none x86_64-unknown-linux-gnu https://github.com/k2-fsa/sherpa-onnx/releases/download/v1.10.28/sherpa-onnx-v1.10.28-linux-x64-static.tar.bz2 sherpa-onnx-v1.10.28-linux-x64-static F93B03D9D9EFB866CE55802EBE4DE413A5FFC46577E42F1167FC17B90212057E 225 +none aarch64-apple-darwin https://github.com/k2-fsa/sherpa-onnx/releases/download/v1.10.28/sherpa-onnx-v1.10.28-osx-universal2-static.tar.bz2 sherpa-onnx-v1.10.28-osx-universal2-static 15D6988A1AA2D32BB82CEE4C3AC13705F9EB606168919B6BBC69099D8100CC14 300 +none x86_64-apple-darwin https://github.com/k2-fsa/sherpa-onnx/releases/download/v1.10.28/sherpa-onnx-v1.10.28-osx-universal2-static.tar.bz2 sherpa-onnx-v1.10.28-osx-universal2-static 15D6988A1AA2D32BB82CEE4C3AC13705F9EB606168919B6BBC69099D8100CC14 300 diff --git a/sys/src/internal/dirs.rs b/sys/src/internal/dirs.rs new file mode 100644 index 0000000..0bbeb05 --- /dev/null +++ b/sys/src/internal/dirs.rs @@ -0,0 +1,206 @@ +// based on https://github.com/pykeio/ort/blob/main/ort-sys/src/internal/dirs.rs and https://github.com/dirs-dev/dirs-sys-rs/blob/main/src/lib.rs + +pub const SHERPA_ROOT: &str = "sherpa-rs"; + +#[cfg(all(target_os = "windows", target_arch = "x86"))] +macro_rules! win32_extern { + ($library:literal $abi:literal $($link_name:literal)? $(#[$doc:meta])? fn $($function:tt)*) => ( + #[link(name = $library, kind = "raw-dylib", modifiers = "+verbatim", import_name_type = "undecorated")] + extern $abi { + $(#[$doc])? + $(#[link_name=$link_name])? + fn $($function)*; + } + ) +} +#[cfg(all(target_os = "windows", not(target_arch = "x86")))] +macro_rules! win32_extern { + ($library:literal $abi:literal $($link_name:literal)? $(#[$doc:meta])? fn $($function:tt)*) => ( + #[link(name = $library, kind = "raw-dylib", modifiers = "+verbatim")] + extern "C" { + $(#[$doc])? + $(#[link_name=$link_name])? + fn $($function)*; + } + ) +} + +#[cfg(target_os = "windows")] +#[allow(non_camel_case_types, clippy::upper_case_acronyms)] +mod windows { + use std::{ + ffi::{c_void, OsString}, + os::windows::prelude::OsStringExt, + path::PathBuf, + ptr, slice, + }; + + #[repr(C)] + #[derive(Clone, Copy)] + struct GUID { + data1: u32, + data2: u16, + data3: u16, + data4: [u8; 8], + } + + impl GUID { + pub const fn from_u128(uuid: u128) -> Self { + Self { + data1: (uuid >> 96) as u32, + data2: (uuid >> 80 & 0xffff) as u16, + data3: (uuid >> 64 & 0xffff) as u16, + #[allow(clippy::cast_possible_truncation)] + data4: (uuid as u64).to_be_bytes(), + } + } + } + + type HRESULT = i32; + type PWSTR = *mut u16; + type PCWSTR = *const u16; + type HANDLE = isize; + type KNOWN_FOLDER_FLAG = i32; + + win32_extern!("SHELL32.DLL" "system" fn SHGetKnownFolderPath(rfid: *const GUID, dwflags: KNOWN_FOLDER_FLAG, htoken: HANDLE, ppszpath: *mut PWSTR) -> HRESULT); + win32_extern!("KERNEL32.DLL" "system" fn lstrlenW(lpstring: PCWSTR) -> i32); + win32_extern!("OLE32.DLL" "system" fn CoTaskMemFree(pv: *const ::core::ffi::c_void) -> ()); + + fn known_folder(folder_id: GUID) -> Option { + unsafe { + let mut path_ptr: PWSTR = ptr::null_mut(); + let result = SHGetKnownFolderPath(&folder_id, 0, HANDLE::default(), &mut path_ptr); + if result == 0 { + let len = lstrlenW(path_ptr) as usize; + let path = slice::from_raw_parts(path_ptr, len); + let ostr: OsString = OsStringExt::from_wide(path); + CoTaskMemFree(path_ptr as *const c_void); + Some(PathBuf::from(ostr)) + } else { + CoTaskMemFree(path_ptr as *const c_void); + None + } + } + } + + #[allow(clippy::unusual_byte_groupings)] + const FOLDERID_LOCAL_APP_DATA: GUID = GUID::from_u128(0xf1b32785_6fba_4fcf_9d557b8e7f157091); + + #[must_use] + pub fn known_folder_local_app_data() -> Option { + known_folder(FOLDERID_LOCAL_APP_DATA) + } +} +#[cfg(target_os = "windows")] +#[must_use] +pub fn cache_dir() -> Option { + self::windows::known_folder_local_app_data().map(|h| h.join(SHERPA_ROOT)) +} + +#[cfg(unix)] +#[allow(non_camel_case_types)] +mod unix { + use std::{ + env, + ffi::{c_char, c_int, c_long, CStr, OsString}, + mem, + os::unix::prelude::OsStringExt, + path::PathBuf, + ptr, + }; + + type uid_t = u32; + type gid_t = u32; + type size_t = usize; + #[repr(C)] + struct passwd { + pub pw_name: *mut c_char, + pub pw_passwd: *mut c_char, + pub pw_uid: uid_t, + pub pw_gid: gid_t, + pub pw_gecos: *mut c_char, + pub pw_dir: *mut c_char, + pub pw_shell: *mut c_char, + } + + extern "C" { + fn sysconf(name: c_int) -> c_long; + fn getpwuid_r( + uid: uid_t, + pwd: *mut passwd, + buf: *mut c_char, + buflen: size_t, + result: *mut *mut passwd, + ) -> c_int; + fn getuid() -> uid_t; + } + + const SC_GETPW_R_SIZE_MAX: c_int = 70; + + #[must_use] + #[cfg(target_os = "linux")] + pub fn is_absolute_path(path: OsString) -> Option { + let path = PathBuf::from(path); + if path.is_absolute() { + Some(path) + } else { + None + } + } + + #[cfg(not(target_os = "windows"))] + #[must_use] + pub fn home_dir() -> Option { + return env::var_os("HOME") + .and_then(|h| if h.is_empty() { None } else { Some(h) }) + .or_else(|| unsafe { fallback() }) + .map(PathBuf::from); + + #[cfg(any(target_os = "android", target_os = "ios", target_os = "emscripten"))] + unsafe fn fallback() -> Option { + None + } + #[cfg(not(any(target_os = "android", target_os = "ios", target_os = "emscripten")))] + unsafe fn fallback() -> Option { + let amt = match sysconf(SC_GETPW_R_SIZE_MAX) { + n if n < 0 => 512, + n => n as usize, + }; + let mut buf = Vec::with_capacity(amt); + let mut passwd: passwd = mem::zeroed(); + let mut result = ptr::null_mut(); + match getpwuid_r( + getuid(), + &mut passwd, + buf.as_mut_ptr(), + buf.capacity(), + &mut result, + ) { + 0 if !result.is_null() => { + let ptr = passwd.pw_dir as *const _; + let bytes = CStr::from_ptr(ptr).to_bytes(); + if bytes.is_empty() { + None + } else { + Some(OsStringExt::from_vec(bytes.to_vec())) + } + } + _ => None, + } + } + } +} + +#[cfg(target_os = "linux")] +#[must_use] +pub fn cache_dir() -> Option { + std::env::var_os("XDG_CACHE_HOME") + .and_then(self::unix::is_absolute_path) + .or_else(|| self::unix::home_dir().map(|h| h.join(".cache").join(SHERPA_ROOT))) +} + +#[cfg(target_os = "macos")] +#[must_use] +pub fn cache_dir() -> Option { + self::unix::home_dir().map(|h| h.join("Library/Caches").join(SHERPA_ROOT)) +} diff --git a/sys/src/internal/mod.rs b/sys/src/internal/mod.rs new file mode 100644 index 0000000..a90ec0a --- /dev/null +++ b/sys/src/internal/mod.rs @@ -0,0 +1 @@ +pub mod dirs; \ No newline at end of file