From 4bd58e64c2df4e719aab8b26f81c2986feabfa15 Mon Sep 17 00:00:00 2001 From: Miroito Date: Thu, 9 Feb 2023 17:02:36 +0100 Subject: [PATCH] feat/Docker deployment --- Cargo.lock | 18 +++++ client/Dockerfile | 91 +++++++++++++++++++++++++ client/src/templates/index.rs | 1 - docker-compose.yml | 50 ++++++++++++++ makefile | 1 + server/Cargo.toml | 2 +- server/Dockerfile | 40 +++++++++++ server/src/env.rs | 75 +++++++++++++++++--- server/src/lib.rs | 1 - server/src/task/get_amf_transactions.rs | 7 ++ 10 files changed, 273 insertions(+), 13 deletions(-) create mode 100644 client/Dockerfile create mode 100644 docker-compose.yml create mode 100644 server/Dockerfile diff --git a/Cargo.lock b/Cargo.lock index 5ce1f84..8db452f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1552,6 +1552,19 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-rustls" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1788965e61b367cd03a62950836d5cd41560c3577d90e40e0819373194d1661c" +dependencies = [ + "http", + "hyper 0.14.24", + "rustls", + "tokio", + "tokio-rustls", +] + [[package]] name = "hyper-tls" version = "0.5.0" @@ -2789,6 +2802,7 @@ dependencies = [ "http", "http-body", "hyper 0.14.24", + "hyper-rustls", "hyper-tls", "ipnet", "js-sys", @@ -2798,16 +2812,20 @@ dependencies = [ "once_cell", "percent-encoding 2.2.0", "pin-project-lite", + "rustls", + "rustls-pemfile", "serde", "serde_json", "serde_urlencoded", "tokio", "tokio-native-tls", + "tokio-rustls", "tower-service", "url 2.3.1", "wasm-bindgen", "wasm-bindgen-futures", "web-sys", + "webpki-roots", "winreg", ] diff --git a/client/Dockerfile b/client/Dockerfile new file mode 100644 index 0000000..f956ab1 --- /dev/null +++ b/client/Dockerfile @@ -0,0 +1,91 @@ +# get the base image +FROM rust:1.66-slim AS build + +# install build dependencies +RUN apt update \ + && apt install -y --no-install-recommends lsb-release apt-transport-https \ + build-essential curl wget pkg-config + +# vars +ENV PERSEUS_VERSION=0.3.6 \ + PERSEUS_SIZE_OPT_VERSION=0.1.9 \ + ESBUILD_VERSION=0.14.7 \ + BINARYEN_VERSION=104 + +# prepare root project dir +WORKDIR /app + +# download the target for wasm +RUN rustup target add wasm32-unknown-unknown + +# install wasm-pack +RUN cargo install wasm-pack + +# retrieve the src dir +COPY . . +# RUN curl https://git.albv.org/alban/fast-insiders/archive/master.tar.gz | tar -xz + +# go to src dir +WORKDIR /app/client + +# install perseus-cli +RUN cargo install perseus-cli --version $PERSEUS_VERSION + +# clean and prep app +RUN perseus clean && perseus prep + +# specify deps in app config +RUN sed -i s"/perseus = .*/perseus = \"${PERSEUS_VERSION}\"/" ./Cargo.toml \ + && sed -i s"/perseus-size-opt = .*/perseus-size-opt = \"${PERSEUS_SIZE_OPT_VERSION}\"/" ./Cargo.toml \ + && cat ./Cargo.toml + +# modify lib.rs +RUN sed -i s'/SizeOpts::default()/SizeOpts { wee_alloc: true, lto: true, opt_level: "s".to_string(), codegen_units: 1, enable_fluent_bundle_patch: false, }/' ./src/lib.rs \ + && cat ./src/lib.rs + +ARG API_URL +ENV API_URL $API_URL + +# run plugin(s) to adjust app +RUN perseus tinker \ + && cat .perseus/Cargo.toml \ + && cat ./src/lib.rs + +# single-threaded perseus CLI mode required for low memory environments +#ENV PERSEUS_CLI_SEQUENTIAL=true + +# deploy app +RUN perseus deploy + +# go back to app dir +WORKDIR /app + +# download and unpack esbuild +RUN curl -O https://registry.npmjs.org/esbuild-linux-64/-/esbuild-linux-64-${ESBUILD_VERSION}.tgz \ + && tar xf esbuild-linux-64-${ESBUILD_VERSION}.tgz \ + && ./package/bin/esbuild --version + +# run esbuild against bundle.js +RUN ./package/bin/esbuild ./client/pkg/dist/pkg/perseus_engine.js --minify --target=es6 --outfile=./client/pkg/dist/pkg/perseus_engine.js --allow-overwrite \ + && ls -lha ./client/pkg/dist/pkg + +# download and unpack binaryen +RUN wget -nv https://github.com/WebAssembly/binaryen/releases/download/version_${BINARYEN_VERSION}/binaryen-version_${BINARYEN_VERSION}-x86_64-linux.tar.gz \ + && tar xf binaryen-version_${BINARYEN_VERSION}-x86_64-linux.tar.gz \ + && ./binaryen-version_${BINARYEN_VERSION}/bin/wasm-opt --version + +# run wasm-opt against bundle.wasm +RUN ./binaryen-version_${BINARYEN_VERSION}/bin/wasm-opt -Os ./client/pkg/dist/pkg/perseus_engine_bg.wasm -o ./client/pkg/dist/pkg/perseus_engine_bg.wasm \ + && ls -lha ./client/pkg/dist/pkg + +# prepare deployment image +FROM debian:stable-slim + +WORKDIR /app + +COPY --from=build /app/client/pkg /app/ + +ENV HOST=0.0.0.0 + +CMD ./server + diff --git a/client/src/templates/index.rs b/client/src/templates/index.rs index cf44c80..726ac53 100644 --- a/client/src/templates/index.rs +++ b/client/src/templates/index.rs @@ -11,7 +11,6 @@ use crate::{ base_button::{BaseButton, BaseButtonStateRx}, paginated_data_table::{PaginatedTable, PaginatedTableStateRx}, }, - env::Config, global_state::AppStateRx, }; diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..e760152 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,50 @@ +version: '3' +services: + client: + container_name: fast-insiders-client + image: fast-insiders-client:latest + build: + context: . + args: + API_URL: http://localhost:8000/v1/ + dockerfile: ./client/Dockerfile + restart: always + ports: + - 8080:8080 + server: + container_name: fast-insiders-server + image: fast-insiders-server:latest + build: + context: . + args: + API_URL: http://localhost:8000/v1/ + dockerfile: ./server/Dockerfile + restart: always + ports: + - 8000:8000 + links: + - db + environment: + - MYSQL_USER=fiuser + - MYSQL_PASSWORD=root + - MYSQL_HOST=db + - MYSQL_DATABASE=fast_insiders + - RUST_LOG=info,lopdf=error + db: + image: mariadb:10.5 + restart: always + command: --transaction-isolation=READ-COMMITTED --binlog-format=ROW + volumes: + - db:/var/lib/mysql + environment: + - MYSQL_ROOT_PASSWORD=root + - MYSQL_PASSWORD=root + - MYSQL_DATABASE=fast_insiders + - MYSQL_USER=fiuser + + +volumes: + fi-data: + driver: local + db: + diff --git a/makefile b/makefile index a5ce10d..aa1659f 100644 --- a/makefile +++ b/makefile @@ -1,5 +1,6 @@ include .env export API_URL # So the client can use it +export RUST_LOG # Migrations should be written first and the model files can be created using this command db_entities: diff --git a/server/Cargo.toml b/server/Cargo.toml index ea850b0..ec74d3c 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -14,7 +14,7 @@ serde_json = { workspace = true } dotenvy = { workspace = true } envy = { workspace = true } tokio = { version = "^1.20.1", features = ["full"] } -reqwest = { version = "0.11", features = ["json"] } +reqwest = { version = "0.11", features = ["json", "rustls-tls"] } rocket = { version = "0.5.0-rc.2", features = ["json"] } sea-orm = { version = "0.10.7", features = [ "runtime-tokio-rustls", diff --git a/server/Dockerfile b/server/Dockerfile new file mode 100644 index 0000000..c58beec --- /dev/null +++ b/server/Dockerfile @@ -0,0 +1,40 @@ +FROM rust:1.66-slim as build + +# Install build dependencies +RUN apt update \ + && apt install -y --no-install-recommends lsb-release apt-transport-https \ + build-essential curl wget pkg-config libssl-dev + +# Root of the project +WORKDIR /app + +# retrieve the src dir +COPY . . +# RUN curl https://git.albv.org/alban/fast-insiders/archive/master.tar.gz | tar -xz + +# go to src dir +# WORKDIR /app + +# Rocket only runs on Nightly +RUN rustup default nightly + +ARG API_URL +ENV API_URL=$API_URL + +# Build the final binary +RUN cargo build --release --bin server + +# prepare deployment image +FROM debian:stable-slim + +# For tls to work +RUN apt-get update && apt-get -y install ca-certificates libssl-dev && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY --from=build /app/target/release/server /app/ + +ENV ROCKET_ADDRESS=0.0.0.0 + +CMD ./server + diff --git a/server/src/env.rs b/server/src/env.rs index a5c8b31..2a482ad 100644 --- a/server/src/env.rs +++ b/server/src/env.rs @@ -1,10 +1,14 @@ -use std::path::PathBuf; - use serde::Deserialize; #[derive(Deserialize, Debug)] -pub struct Config { - pub database_url: String, +pub struct Env { + pub mysql_user: String, + pub mysql_password: String, + pub mysql_host: String, + #[serde(default = "mysql_port_default")] + pub mysql_port: String, + #[serde(default = "mysql_database_default")] + pub mysql_database: String, #[serde(default = "max_connections_default")] pub max_connections: u32, #[serde(default = "min_connections_default")] @@ -27,6 +31,14 @@ pub struct Config { pub get_amf_transaction_interval: u64, } +fn mysql_port_default() -> String { + "3306".to_string() +} + +fn mysql_database_default() -> String { + "fast-insiders".to_string() +} + fn max_connections_default() -> u32 { 100 } @@ -67,19 +79,62 @@ fn get_amf_transaction_interval() -> u64 { 3600 } +impl Env { + fn new() -> Self { + if cfg!(debug_assertions) { + // A .env file will only be loaded in a dev or debug environment + dotenvy::dotenv().expect("Failed to load .env file"); + } + + let env = envy::from_env::().expect("Failed to load env"); + + env + } +} + +#[derive(Debug)] +pub struct Config { + pub database_url: String, + pub max_connections: u32, + pub min_connections: u32, + pub connect_timeout: u64, + pub acquire_timeout: u64, + pub idle_timeout: u64, + pub max_lifetime: u64, + pub sqlx_logging: bool, + pub amf_informations_req: String, + pub amf_documents_path: String, + pub get_amf_transaction_interval: u64, +} + impl Config { pub fn new() -> Self { - dotenvy::dotenv().expect("Failed to load .env file"); - let mut config = envy::from_env::().expect("Failed to load env"); + let env = Env::new(); + let database_url = format!( + "mysql://{}:{}@{}:{}/{}", + env.mysql_user, env.mysql_password, env.mysql_host, env.mysql_port, env.mysql_database + ); + + let mut config = Config { + database_url, + max_connections: env.max_connections, + min_connections: env.min_connections, + connect_timeout: env.connect_timeout, + acquire_timeout: env.acquire_timeout, + idle_timeout: env.idle_timeout, + max_lifetime: env.max_lifetime, + sqlx_logging: env.sqlx_logging, + amf_informations_req: env.amf_informations_req, + amf_documents_path: env.amf_documents_path, + get_amf_transaction_interval: env.get_amf_transaction_interval, + }; if config.amf_documents_path.chars().last().unwrap_or('/') != '/' { config.amf_documents_path.push('/'); } + info!("Config: {:?}", config); + config } } - -pub fn load_env() -> Result { - dotenvy::dotenv() -} diff --git a/server/src/lib.rs b/server/src/lib.rs index 4e72674..6e3031d 100644 --- a/server/src/lib.rs +++ b/server/src/lib.rs @@ -68,7 +68,6 @@ async fn start_rocket() -> Result<(), sea_orm_rocket::rocket::Error> { #[rocket::main] pub async fn main() -> Result<(), Box> { - env::load_env()?; logger::init_log()?; // Run tasks diff --git a/server/src/task/get_amf_transactions.rs b/server/src/task/get_amf_transactions.rs index 2b78df4..6b5ca2e 100644 --- a/server/src/task/get_amf_transactions.rs +++ b/server/src/task/get_amf_transactions.rs @@ -45,8 +45,15 @@ impl GetAMFTransactions { info!("Starting AMF transaction download task"); let mut from = 0; + let req = AMFRequest::new(AMFRequestType::DD, 0, 10); + match req.get_list().await { + Ok(_) => (), + Err(e) => warn!("The AMF transaction public api is not available: {}", e), + }; + let mut req = AMFRequest::new(AMFRequestType::DD, from, self.max_req_size); let mut tr_to_process = Vec::new(); + 'outer: while let Some(resp) = req.get_list().await.ok() { info!( "Downloading hit list from {} to {}", -- 2.36.3