books/manual/SUMMARY.md

@@ -47,6 +47,7 @@
     - [TFChain](dashboard/tfchain/tfchain.md)
       - [Your Profile](dashboard/tfchain/your_profile.md)
       - [TF DAO](dashboard/tfchain/tf_dao.md)
+        - [Voting Weight Example](dashboard/tfchain/tf_dao_voting_weight.md)
       - [TF Token Bridge](dashboard/tfchain/tf_token_bridge.md)
       - [TF Token Transfer](dashboard/tfchain/tf_token_transfer.md)
       - [TF Minting Reports](dashboard/tfchain/tf_minting_reports.md)
@@ -117,13 +118,14 @@
     - [Internals](developers/internals/internals.md)
       - [Reliable Message Bus - RMB](developers/internals/rmb/rmb_toc.md)
         - [Introduction to RMB](developers/internals/rmb/rmb_intro.md)
+        - [Coding Guides](developers/internals/rmb/rmb_coding_guides.md)
         - [RMB Specs](developers/internals/rmb/rmb_specs.md)
         - [RMB Peer](developers/internals/rmb/uml/peer.md)
         - [RMB Relay](developers/internals/rmb/uml/relay.md)
       - [Zero-OS](developers/internals/zos/zos_readme.md)
         - [Manual](developers/internals/zos/manual/manual.md)
         - [Workload Types](developers/internals/zos/manual/workload_types.md)
-        - [Internal Modules](developers/internals/zos/internals/internals.md)
+        - [Internal Modules](developers/internals/zos/internals/zos_internals.md)
           - [Identity](developers/internals/zos/internals/identity/identity_readme.md)
             - [Node ID Generation](developers/internals/zos/internals/identity/identity.md)
             - [Node Upgrade](developers/internals/zos/internals/identity/upgrade.md)

collections/dashboard/tfchain/tf_dao.md

@@ -53,4 +53,6 @@ The DAO votes are weighted as follows:
   2 * (sum of CU of all nodes) + (sum of SU of all nodes)
   ```
 
-Voting weights are tracked per farm to keep it easy and traceable. Thus, if an account has multiple farms, the vote will be registered per farm.
+Voting weights are tracked per farm to keep it easy and traceable. Thus, if an account has multiple farms, the vote will be registered per farm.
+
+For more information on voting weight, [check this example](./tf_dao_voting_weight.md).

collections/dashboard/tfchain/tf_dao_voting_weight.md

@@ -0,0 +1,90 @@
+<h1> Voting Weight Example</h1>
+
+<h2>Table of Contents</h2>
+
+- [Introduction](#introduction)
+- [Example](#example)
+
+---
+
+## Introduction
+
+As we've seen in the previous section, voting weights for the TF DAO are tracked per farm to keep it easy and traceable.
+
+Let's see an example of how the weight of a farm is calculated.
+
+## Example
+
+For this example, we will take a node from QANet:
+
+```
+node_id = 3
+farm_id = 1
+```
+
+The node information is the following:
+
+![Alt text](../img/tf_dao_voting_weight_example.png)
+
+The Rust code that calculates node weight extracted from TFChain can be found [here](https://github.com/threefoldtech/tfchain/blob/eb36aa90df2d60cb1a534997903821fc68a096f1/substrate-node/support/src/resources.rs#L57-L91). 
+
+When we apply this code to the current node, we get the following:
+
+```
+const GIGABYTE: u128 = 1024 * 1024 * 1024;
+const ONE_THOUSAND: u128 = 1000;
+const CRU: u64 = 24;
+const MRU: u64 = 202803462144;
+const SRU: u64 = 512110190592;
+const HRU: u64 = 9001778946048;
+fn calc_cu() -> u64 {
+    let cru_min = CRU as u128 * 2 * GIGABYTE * ONE_THOUSAND;
+    let mru_min =
+        ((MRU as u128).checked_sub(1).unwrap_or(0) * GIGABYTE) * ONE_THOUSAND / 4;
+    let sru_min = SRU as u128 * ONE_THOUSAND / 50;
+    if cru_min < mru_min && cru_min < sru_min {
+        cru_min as u64
+    } else if mru_min < cru_min && mru_min < sru_min {
+        mru_min as u64
+    } else if sru_min < cru_min && sru_min < mru_min {
+        sru_min as u64
+    } else {
+        0
+    }
+}
+    
+fn get_cu() -> u64 {
+    let cu = calc_cu();
+    let calculated_cu = 2 * (cu as u128 / GIGABYTE / ONE_THOUSAND);
+    calculated_cu as u64
+}
+    
+fn get_su() -> u64 {
+    let su = HRU as u128 * ONE_THOUSAND / 1200 + SRU as u128 * ONE_THOUSAND / 250;
+    let calculated_su = su / GIGABYTE;
+    let result = calculated_su as u128 / ONE_THOUSAND;
+    result as u64
+}
+fn main() {
+    println!("CU: {}", get_cu());
+    println!("SU: {}", get_su());
+}
+```  
+
+We can then simply paste this code in the [Rust playground](https://play.rust-lang.org/) to obtain the `CU` and `SU` values of this node:
+
+``` 
+CU: 18
+SU: 8
+```
+
+We can then simply put these numbers in the [equation shown in the last section](./tf_dao.md#voting-weight) to obtain the farm weight. Note that in this case, the farm has 2 different nodes with the same `CU` and `SU` values.
+
+``` 
+Weight = 2 * (sum of CU of all nodes) + (sum of SU of all nodes)
+Weight = 2 * (18+18) + (8+8)
+Weight = 2 * 36 + 16
+Weight = 88
+```
+
+We can see that the farm has the weight `88`.

collections/developers/developers.md

@@ -76,7 +76,7 @@ For complementary information on the technology developed by ThreeFold, refer to
   - [ZOS](index.md)
     - [Manual](manual.md)
     - [Workload Types](workload_types.md)
-    - [Internal Modules](internals.md)
+    - [Internal Modules](zos_internals.md)
     - [Capacity](capacity.md)
     - [Performance Monitor Package](performance.md)
       - [Public IPs Validation Task](publicips.md)

collections/developers/flist/flist_case_studies/flist_nextcloud_case_study.md

@@ -819,8 +819,8 @@ output "wg_config" {
 output "vm1_ip" {
   value = grid_deployment.d1.vms[0].ip
 }
-output "vm1_ygg_ip" {
-  value = grid_deployment.d1.vms[0].ygg_ip
+output "vm1_planetary_ip" {
+  value = grid_deployment.d1.vms[0].planetary_ip
 }
 
 output "fqdn" {

collections/developers/internals/internals.md

@@ -6,6 +6,7 @@ We present in this section of the developers book a partial list of system compo
 
 - [Reliable Message Bus - RMB](rmb_toc.md)
   - [Introduction to RMB](rmb_intro.md)
+  - [Coding Guides](rmb_coding_guides.md)
   - [RMB Specs](rmb_specs.md)
   - [RMB Peer](peer.md)
   - [RMB Relay](relay.md)
@@ -13,7 +14,7 @@ We present in this section of the developers book a partial list of system compo
 - [ZOS](zos_readme.md)
   - [Manual](manual.md)
   - [Workload Types](workload_types.md)
-  - [Internal Modules](internals.md)
+  - [Internal Modules](zos_internals.md)
   - [Capacity](capacity.md)
   - [Performance Monitor Package](performance.md)
   - [API](api.md)

collections/developers/internals/rmb/rmb_coding_guides.md

@@ -0,0 +1,174 @@
+<h1> Coding Guides </h1>
+
+<h2> Table of Contents </h2>
+
+- [Introduction](#introduction)
+- [Module Structure](#module-structure)
+  - [Example](#example)
+- [Naming Conventions](#naming-conventions)
+- [`error` Handling](#error-handling)
+- [`logs`](#logs)
+- [Function Signatures](#function-signatures)
+  - [Examples](#examples)
+
+---
+
+## Introduction
+
+We present coding guides for the reliable message bus (RMB).
+
+> This document will always be `work in progress`. [Read the official docs for updates](https://github.com/threefoldtech/rmb-rs/blob/main/docs/code-guide.md).
+## Module Structure
+
+In Rust there are multiple ways to create a (sub)module in your crate
+
+- `<module>.rs`  A single file module. can be imported in `main.rs` or `lib.rs` with the keyword `mod`
+- `<module>/mod.rs` A directory module. Uses mod.rs as the module `entrypoint` always. Other sub-modules can be created next to mod.rs and can be made available by using the `mod` keyword again in the `mod.rs` file.
+
+We will agree to use the 2nd way (directory) but with the following restrictions:
+
+- `mod.rs` will have all `traits` and `concrete types` used by the traits.
+- `<implementation>.rs` file next to `mod.rs` that can include implementation for the module trait.
+z
+
+### Example
+
+Following is an example of `animal` module.
+
+```
+animal/
+   mod.rs
+   dog.rs
+   cat.rs
+```
+
+> File names are always in `snake_case` but avoid the `_` as much as possible because they basically look ugly in file tree. For example we prefer the name `dog.rs` over `dog_animal.rs` because we already can tell from the module name that it's a `dog` __animal__. Hence in the identity module for example the name `ed25519.rs` is preferred over `ed25519_identity.rs` because that's already inferred from the module name.
+The `mod.rs` file then can contain
+
+```rust
+pub mod dog;
+pub mod cat;
+pub use dog::Dog;
+
+pub trait Food {
+    fn calories(&self) -> u32;
+}
+
+pub trait Animal<F>
+where
+    F: Food,
+{
+    fn feed(&mut self, food: F);
+}
+
+```
+
+The `dog.rs` file then can contain
+
+```rust
+use super::{Animal, Food};
+
+pub struct DogFood {}
+impl Food for DogFood {
+    fn calories(&self) -> u32 {
+        1000
+    }
+}
+pub struct Dog {}
+
+impl Animal<DogFood> for Dog {
+    fn feed(&mut self, food: DogFood) {
+        println!("yum yum yum {} calories", food.calories());
+    }
+}
+
+```
+
+A user of the module now can do
+
+```
+use animal::dog::{Dog, DogFood};
+```
+
+For common implementation that are usually used in your modules, a `pub use` can be added in `mod.rs` to make it easier to import your type. For example
+
+```rust
+// dog is brought directly from animal crate
+use animal::Dog;
+// cat i need to get from the sub-module
+use animal::cat::Cat;
+```
+
+## Naming Conventions
+
+Following the rust guide lines for name
+
+- `file names` are short snake case. avoid `_` if otherwise name will not be descriptive. Check note about file names above.
+- `trait`, `struct`, `enum` names are all `CamelCase`
+- `fn`, `variables` names are snake case
+
+Note, names of functions and variables need to be `descriptive` but __short__ at the same time. Also avoid the `_` until absolutely necessary. A variable with a single `word` name is better if it doesn't cause confusion with other variables in the same context.
+
+The name of the variable should never include the `type`.
+
+## `error` Handling
+
+We agreed to use `anyhow` crate in this project. Please read the docs for [`anyhow`](https://docs.rs/anyhow/1.0.57/anyhow/)
+
+To unify the practice by default we import both `Result` and `Context` from `anyhow`
+> Others can be imported as well if needed.
+```rust
+use anyhow::{Result, Context};
+
+fn might_fail() -> Result<()> {
+    // context adds a `context` to the error. so if another_call fails. I can tell exactly failed when i was doing what
+    another_call().context("failed to do something")?; // <- we use ? to propagate the error unless you need to handle the error differently
+
+    Ok(()) // we use Ok from std no need to import anyhow::Ok although it's probably the same.
+}
+
+fn might_fail2() -> Result<()> {
+    if fail {
+        // use the bail macro fom anyhow to exit with an error.
+        bail!("failed because fail with set to true");
+    }
+}
+
+> NOTE: all error messages starts with lowercase. for example it's `failed to ...` not `Failed to ...`
+```
+
+## `logs`
+
+logging is important to trace the errors that cannot be propagated and also for debug messages that can help spotting a problem. We always gonna use `log` crate. as
+
+```rust
+log::debug!(); // for debug messages
+log::info!(); // info messages
+```
+
+Note only `errors` that can __NOT__ be propagated are logged.
+
+> NOTE: All log messages start with lowercase.
+## Function Signatures
+
+For function inputs (arguments) `generic` types are preferred if available over concrete types. This most obvious with `string` types. depending on the function behavior
+
+### Examples
+
+This is bad:
+
+```rust
+fn call1(key: String);
+fn call2(key: &str);
+```
+
+It is preferred to use:
+
+```rust
+// in case function will need to take ownership of the string.
+fn call1<k: Into<String>>(k: K);
+// inc ase function will just need to use a reference to the string.
+fn call2<K: AsRef<str>>(k: K);
+
+// this will allow both functions to be callable with `&str`, `String`.
+```

collections/developers/internals/rmb/rmb_intro.md

@@ -2,106 +2,294 @@
 
 <h2> Table of Contents </h2>
 
-- [What is RMB](#what-is-rmb)
-- [Why](#why)
-- [Specifications](#specifications)
-- [How to Use RMB](#how-to-use-rmb)
-- [Libraries](#libraries)
-  - [Known Libraries](#known-libraries)
-  - [No Known Libraries](#no-known-libraries)
-- [What is rmb-peer](#what-is-rmb-peer)
-- [Download](#download)
-- [Building](#building)
-- [Running tests](#running-tests)
+- [Introduction](#introduction)
+- [Overview of the Operation of RMB Relay](#overview-of-the-operation-of-rmb-relay)
+  - [Connections](#connections)
+  - [Federations](#federations)
+  - [Peer](#peer)
+  - [Peer Implementation](#peer-implementation)
+    - [`rmb-peer` message types](#rmb-peer-message-types)
+      - [Output Requests](#output-requests)
+      - [Incoming Response](#incoming-response)
+      - [Incoming Request](#incoming-request)
+      - [Outgoing Response](#outgoing-response)
+- [End2End Encryption](#end2end-encryption)
+  - [Rate Limiting](#rate-limiting)
+  - [Substrate Connections](#substrate-connections)
+  - [Redundancy and Failover](#redundancy-and-failover)
 
----
+***
 
-## What is RMB
+## Introduction
 
-Reliable message bus is a secure communication panel that allows `bots` to communicate together in a `chat` like way. It makes it very easy to host a service or a set of functions to be used by anyone, even if your service is running behind NAT.
+RMB is (reliable message bus) is a set of protocols and a `relay` server that aims to abstract inter-process communication between multiple processes running over multiple nodes.
 
-Out of the box RMB provides the following:
+The point behind using RMB is to allow the clients to not know much about the other process, or where it lives (client doesn't know network addresses, or identity). Unlike HTTP(S) where the caller must know exact address (or dns-name) and endpoints of the calls. Instead RMB requires you to only know about
 
-- Guarantee authenticity of the messages. You are always sure that the received message is from whoever is pretending to be
-- End to End encryption
-- Support for 3rd party hosted relays. Anyone can host a relay and people can use it safely since there is no way messages can be inspected while using e2e. That's similar to `home` servers by `matrix`
+- Twin ID (numeric ID) of where the service can be found
+- Command (string) is simply the function to call
+- The request "body" which is binary blob that is passed to the command as is
+  - implementation of the command need then to interpret this data as intended (out of scope of rmb)
 
-![layout](img/layout.png)
+Twins are stored on tfchain. hence identity of twins is granted not to be spoofed, or phished. When a twin is created he needs to define 2 things:
 
-## Why
+- RMB RELAYS
+- His Elliptic Curve public key (we use secp256k1 (K-256) elliptic curve)
 
-RMB is developed by ThreefoldTech to create a global network of nodes that are available to host capacity. Each node will act like a single bot where you can ask to host your capacity. This enforced a unique set of requirements:
+Once all twins has their data set correctly on the chain. Any 2 twins can communicate with full end-to-end encryption as follows:
 
-- Communication needed to be reliable
-  - Minimize and completely eliminate message loss
-  - Reduce downtime
-- Node need to authenticate and authorize calls
-  - Guarantee identity of the other peer so only owners of data can see it
-- Fast request response time
+- A twin establish a WS connection to his relays
+- A twin create an `envelope` as defined by the protobuf schema
+- Twin fill end all envelope information (more about this later)
+- Twin pushes the envelope to one of his relays
+When received by the relay, it will determine whether to deliver the message to one of its directly connected clients or to a remote relay. the message processed as follows:
+  - If the destination twin is also using the same relay, local route will have priority over others and message is directly forwarded to this twin.
+  - If federation is needed (twin using different relay), message is forwarded to one of the twin's relays.
 
-Starting from this we came up with a more detailed requirements:
+Any new messages that is designated to this twin, is pushed over the websocket to this twin. The twin can NOT maintain multiple connections to same relay hence a small tool (rmb-peer) is provided that runs as a dispatcher for a single twin identity.
 
-- User (or rather bots) need their identity maintained by `tfchain` (a blockchain) hence each bot needs an account on tfchain to be able to use `rmb`
-- Then each message then can be signed by the `bot` keys, hence make it easy to verify the identity of the sender of a message. This is done both ways.
-- To support federation (using 3rd party relays) we needed to add e2e encryption to make sure messages that are surfing the public internet can't be sniffed
-- e2e encryption is done by deriving an encryption key from the same identity seed, and share the public key on `tfchain` hence it's available to everyone to use
+This rmb-peer tool makes it possible to run multiple services behind this twin and push replies back to their initiators
 
-## Specifications
+## Overview of the Operation of RMB Relay
 
-For details about protocol itself please check the [specs](rmb_specs.md).
+![](./img/rmb_overview.png)
 
-## How to Use RMB
+### Connections
 
-There are many ways to use `rmb` because it was built for `bots` and software to communicate. Hence, there is no mobile app for it for example, but instead a set of libraries where you can use to connect to the network, make chitchats with other bots then exit.
+The relay can maintain **MULTIPLE** connections per peer given that each connection has a unique **SID** (session id). But for each (twin-id, session-id) combo there can be only one connection. if a new connection with the same (twin-id, session-id) is created, the older connection is dropped.
 
-Or you can keep the connection forever to answer other bots requests if you are providing a service.
+The `rmb-peer` process reserved the `None` sid. It connection with No session id, hence you can only run one `rmb-peer` per `twin` (identity). But the same twin (identity) can make other connection with other rmb clients (for example rmb-sdk-go direct client) to establish more connections with unique session ids.
 
-## Libraries
+### Federations
 
-If there is a library in your preferred language, then you are in luck! Simply follow the library documentations to implement a service bot, or to make requests to other bots.
+Starting from version 1.1.0, the federation field has been deprecated, and the logic of federation has moved to happen in the relays. Relay is now responsible for determining whether to deliver the message to one of its directly connected clients or to a remote relay.
 
-### Known Libraries
+Relay now has an in-memory ranking system to rank its known relays according to their known mean failure rate over a recent configured period of time (time window).
 
-- Golang [rmb-sdk-go](https://github.com/threefoldtech/rmb-sdk-go)
-- Typescript [rmb-sdk-ts](https://github.com/threefoldtech/rmb-sdk-ts)
+The ranking system used to give a hint to the router (when a twin has multiple configured relays on-chain) to try the relays that have a higher chance of working first, minimizing routing messages to services that failed recently (minimizing the latency).
 
-### No Known Libraries
+The rank of a relay will heal over time because the system will only consider failures in the recent time frame, allowing the router to revisit the relay and distribute the load between all working services. For relays with the same failure rate, the order will be randomized.
 
-If there are no library in your preferred language, here's what you can do:
+The ranker time window can be configured when starting the relay by specifying the period in seconds after the `--ranker_period` option. If the option is omitted, the default value of one hour will be used.
 
-- Implement a library in your preferred language
-- If it's too much to do all the signing, verification, e2e in your language then use `rmb-peer`
+Example:
 
-## What is rmb-peer
-
-think of `rmb-peer` as a gateway that stands between you and the `relay`. `rmb-peer` uses your mnemonics (your identity secret key) to assume your identity and it connects to the relay on your behalf, it maintains the connection forever and takes care of
-
-- reconnecting if connection was lost
-- verifying received messages
-- decrypting received messages
-- sending requests on your behalf, taking care of all crypto heavy lifting.
-
-Then it provide a simple (plain-text) api over `redis`. means to send messages (or handle requests) you just need to be able to push and pop messages from some redis queues. Messages are simple plain text json.
-
-> More details can be found [here](rmb_specs.md)
-
-
-## Download
-
-Please check the latest [releases](https://github.com/threefoldtech/rmb-rs/releases) normally you only need the `rmb-peer` binary, unless you want to host your own relay.
-
-## Building
-
-```bash
-git clone git@github.com:threefoldtech/rmb-rs.git
-cd rmb-rs
-cargo build --release --target=x86_64-unknown-linux-musl
+```
+rmb-relay --substrate wss://tfchain.dev.grid.tf:443 --domain r1.3x0.me --ranker-period 1800
 ```
 
-## Running tests
+### Peer
 
-While inside the repository
+Any language or code that can open `WebSocket` connection to the relay can work as a peer. A peer need to do the following:
+
+- Authenticate with the relay. This is by providing a `JWT` that is signed by the twin key (more on that later)
+- Handle received binary mesasge
+- Send binary messages
+
+Each message is an object of type `Envelope` serialized as with protobuf. Type definition can be found under `proto/types.proto`
+
+### Peer Implementation
+
+This project already have a peer implementation that works as local peer gateway. By running this peer instance it allows you to
+run multiple services (and clients) behind that gateway and they appear to the world as a single twin.
+
+- The peer gateway (rmb-peer) starts and connects to his relays
+- If requests are received, they are verified, decrypted and pushed to a redis queue that as command specific (from the envelope)
+- A service can then be waiting on this redis queue for new messages
+  - The service can process the command, and push a response back to a specific redis queue for responses.
+- The gateway can then pull ready responses from the responses queue, create a valid envelope, encrypt, and sign and send to destination
+
+![](./img/rmb_peer.png)
+
+#### `rmb-peer` message types
+
+To make it easy for apps to work behind an `rmb-peer`, we use JSON message for communication between the local process and the rmb-peer. the rmb-peer still
+maintains a fully binary communication with the relay.
+
+A request message is defined as follows
+
+##### Output Requests
+
+This is created by a client who wants to request make a request to a remote service
+
+> this message is pushed to `msgbus.system.local` to be picked up by the peer
+
+```rust
+#[derive(Serialize, Deserialize, Clone, Debug)]
+pub struct JsonOutgoingRequest {
+    #[serde(rename = "ver")]
+    pub version: usize,
+    #[serde(rename = "ref")]
+    pub reference: Option<String>,
+    #[serde(rename = "cmd")]
+    pub command: String,
+    #[serde(rename = "exp")]
+    pub expiration: u64,
+    #[serde(rename = "dat")]
+    pub data: String,
+    #[serde(rename = "tag")]
+    pub tags: Option<String>,
+    #[serde(rename = "dst")]
+    pub destinations: Vec<u32>,
+    #[serde(rename = "ret")]
+    pub reply_to: String,
+    #[serde(rename = "shm")]
+    pub schema: String,
+    #[serde(rename = "now")]
+    pub timestamp: u64,
+}
+```
+
+##### Incoming Response
+
+A response message is defined as follows this is what is received as a response by a client in response to his outgoing request.
+
+> this response is what is pushed to `$ret` queue defined by the outgoing request, hence the client need to wait on this queue until the response is received or it times out
+
+```rust
+#[derive(Serialize, Deserialize, Clone, Debug)]
+pub struct JsonError {
+    pub code: u32,
+    pub message: String,
+}
+
+#[derive(Serialize, Deserialize, Clone, Debug)]
+pub struct JsonIncomingResponse {
+    #[serde(rename = "ver")]
+    pub version: usize,
+    #[serde(rename = "ref")]
+    pub reference: Option<String>,
+    #[serde(rename = "dat")]
+    pub data: String,
+    #[serde(rename = "src")]
+    pub source: String,
+    #[serde(rename = "shm")]
+    pub schema: Option<String>,
+    #[serde(rename = "now")]
+    pub timestamp: u64,
+    #[serde(rename = "err")]
+    pub error: Option<JsonError>,
+}
+```
+
+##### Incoming Request
+
+An incoming request is a modified version of the request that is received by a service running behind RMB peer
+> this request is received on `msgbus.${request.cmd}` (always prefixed with `msgbus`)
+
+```rust
+#[derive(Serialize, Deserialize, Clone, Debug)]
+pub struct JsonIncomingRequest {
+    #[serde(rename = "ver")]
+    pub version: usize,
+    #[serde(rename = "ref")]
+    pub reference: Option<String>,
+    #[serde(rename = "src")]
+    pub source: String,
+    #[serde(rename = "cmd")]
+    pub command: String,
+    #[serde(rename = "exp")]
+    pub expiration: u64,
+    #[serde(rename = "dat")]
+    pub data: String,
+    #[serde(rename = "tag")]
+    pub tags: Option<String>,
+    #[serde(rename = "ret")]
+    pub reply_to: String,
+    #[serde(rename = "shm")]
+    pub schema: String,
+    #[serde(rename = "now")]
+    pub timestamp: u64,
+}
+```
+
+Services that receive this needs to make sure their responses `destination` to have the same value as the incoming request `source`
+
+##### Outgoing Response
+
+A response message is defined as follows this is what is sent as a response by a service in response to an incoming request.
+
+Your bot (server) need to make sure to set `destination` to the same value as the incoming request `source`
+
+> This response is what is pushed to `msgbus.system.reply`
+
+```rust
+#[derive(Serialize, Deserialize, Clone, Debug)]
+pub struct JsonOutgoingResponse {
+    #[serde(rename = "ver")]
+    pub version: usize,
+    #[serde(rename = "ref")]
+    pub reference: Option<String>,
+    #[serde(rename = "dat")]
+    pub data: String,
+    #[serde(rename = "dst")]
+    pub destination: String,
+    #[serde(rename = "shm")]
+    pub schema: Option<String>,
+    #[serde(rename = "now")]
+    pub timestamp: u64,
+    #[serde(rename = "err")]
+    pub error: Option<JsonError>,
+}
+```
+
+## End2End Encryption
+
+Relay is totally opaque to the messages. Our implementation of the relay does not poke into messages except for the routing attributes (source, and destinations addresses, and federation information). But since the relay is designed to be hosted by other 3rd parties (hence federation) you should
+not fully trust the relay or whoever is hosting it. Hence e2e was needed
+
+As you already understand e2e is completely up to the peers to implement, and even other implementations of the peers can agree on a completely different encryption algorithm and key sharing algorithm (again, relay does not care). But in our implementation of the e2e (rmb-peer) things goes like this
+
+- Each twin has a `pk` field on tfchain. when rmb-peer start, it generates an `secp256k1` key from the same seed as the user tfchain mnemonics. Note that this will not make the encryption key and the signing key any related, they just are driven from the same seed.
+- On start, if the key is not already set on the twin object, the key is updated.
+- If a peer A is trying to send a message to peer B. but peer B does not has his `pk` set, peer A will send the message in plain-text format (please check the protobuf envelope type for details)
+- If peer B has public key set, peer A will prefer e2e encryption and will does the following:
+- Drive a shared secret point with `ecdh` algorithm, the key is the `sha256` of that point
+- `shared = ecdh(A.sk, B.pk)`
+- create a 12 bytes random nonce
+- encrypt data as `encrypted = aes-gcm.encrypt(shared-key, nonce, plain-data)`
+- create cipher as `cipher nonce + encrypted`
+- fill `envelope.cipher = cipher`
+- on receiving a message peer B does the same in the opposite direction
+- split data and nonce (nonce is always first 12 bytes)
+- derive the same shared key
+- `shared = ecdh(B.sk, A.pk)`
+- `plain-data = aes-gcm.decrypt(shared-key, nonce, encrypted)`
+
+### Rate Limiting
+
+To avoid abuse of the server, and prevent DoS attacks on the relay, a rate limiter is used to limit the number of clients' requests.\
+It was decided that the rate limiter should only watch websocket connections of users, since all other requests/connections with users consume little resources, and since the relay handles the max number of users inherently.\
+The limiter's configurations are passed as a command line argument `--limit <count>, <size>`. `<count>` represents the number of messages a twin is allowed to send in each time window, `<size>` represents the total size of messages in bytes a twin is allowed to send in each time window.\
+Currently there are two implementations of the rate limiter:
+
+- `NoLimit` which imposes no limits on users.
+- `FixedWindowLimiter` which breaks the timeline into fixed time windows, and allows a twin to send a fixed number of messages, with a fixed total size, in each time window. If a twin exceeded their limits in some time window, their message is dropped, an error message is sent back to the user, the relay dumps a log about this twin, and the user gets to keep their connection with the relay.
+
+### Substrate Connections
+
+To make the relay and peer more reliable, the substrate client accepts multiple substrate urls.
+> To provide multiple urls use the `--substrate <url>` command line argument multiple times.
+
+Example:
 
 ```bash
-cargo test
+  rmb-peer --substrate wss://tfchain.grid.tf:443 --substrate wss://02.tfchain.grid.tf:443 --substrate wss://03.tfchain.grid.tf:443
 ```
+
+It's important to note the only one substrate client is held at a time, and the other urls are only used in the case of a network failure.\
+This way, if a substrate connection failed, other urls are used to try to connect to substrate.\
+The client uses iterates between urls in a Round Robin fashion, and tries to reconnect. If a specified number of trials is done (currently 2x the number of urls) and none of them was successful, the client fails and returns and error.
+
+### Redundancy and Failover
+
+Starting from version 1.1.0, RMB has integrated redundancy and failover into the system to achieve high availability.
+This is done by allowing RMB-peer to set more than one relay domain for a twin on-chain and establish connections with several redundant relays at the same time.
+
+Enabling failover ensures that communication between twins can continue even if one of the relays fails, as the client will eventually route the message to its destination through another operational relay.
+
+Example:
+
+```bash
+  rmb-peer -m "{MNEMONIC}" --substrate wss://tfchain.dev.grid.tf:443 --relay wss://r1.dev.grid.tf --relay wss://r2.dev.grid.tf
+```

collections/developers/internals/rmb/rmb_toc.md

@@ -13,6 +13,7 @@ Out of the box RMB provides the following:
 <h2> Table of Contents </h2>
 
 - [Introduction to RMB](rmb_intro.md)
+- [Coding Guides](rmb_coding_guides.md)
 - [RMB Specs](rmb_specs.md)
 - [RMB Peer](peer.md)
 - [RMB Relay](relay.md)

collections/developers/internals/rmb/uml/relay.md

@@ -1,4 +1,4 @@
-<h1> RMB Peer </h1>
+<h1> RMB Relay </h1>
 
 <h2> Table of Contents </h2>
 

collections/system_administrators/pulumi/pulumi_deployment_details.md

@@ -277,7 +277,7 @@ resources:
 
 outputs:
   node_deployment_id: ${deployment.node_deployment_id}
-  ygg_ip: ${deployment.vms_computed[0].ygg_ip}
+  planetary_ip: ${deployment.vms_computed[0].planetary_ip}
 ```
 
 We have a scheduler, and a network just like before. But now, we also have a deployment `grid:internal:Deployment` object that can have one or more disks and virtual machines.
@@ -357,7 +357,7 @@ We now see how to deploy a [Kubernetes cluster using Pulumi](https://github.com/
 
 outputs:
   node_deployment_id: ${kubernetes.node_deployment_id}
-  ygg_ip: ${kubernetes.master_computed.ygg_ip}
+  planetary_ip: ${kubernetes.master_computed.planetary_ip}
 ```
 
 Now, we define the Kubernetes resource `grid:internal:Kubernetes` that has master and workers slice. You define almost everything like a normal VM except for the FLiist. Also note that the token is the `cluster token`. This will ensure that the workers and the master communicate properly.
@@ -435,7 +435,7 @@ Here's an [example](https://github.com/threefoldtech/pulumi-provider-grid/blob/d
       node_id: 14
       fqdn: mydomain.com
       backends:
-        - http://[${deployment.vms_computed[0].ygg_ip}]:9000
+        - http://[${deployment.vms_computed[0].planetary_ip}]:9000
 ```
 
 Here, we informed the gateway that any request coming for the domain `mydomain.com` needs to be balanced through the backends.

collections/system_administrators/terraform/advanced/terraform_capacity_planning.md

@@ -77,15 +77,15 @@ resource "grid_deployment" "d1" {
 output "vm1_ip" {
   value = grid_deployment.d1.vms[0].ip
 }
-output "vm1_ygg_ip" {
-  value = grid_deployment.d1.vms[0].ygg_ip
+output "vm1_planetary_ip" {
+  value = grid_deployment.d1.vms[0].planetary_ip
 }
 
 output "vm2_ip" {
   value = grid_deployment.d1.vms[1].ip
 }
-output "vm2_ygg_ip" {
-  value = grid_deployment.d1.vms[1].ygg_ip
+output "vm2_planetary_ip" {
+  value = grid_deployment.d1.vms[1].planetary_ip
 }
 ```
 

collections/system_administrators/terraform/advanced/terraform_mariadb_synced_databases.md

@@ -237,11 +237,11 @@ output "node1_zmachine2_ip" {
   value = grid_deployment.d2.vms[0].ip
 }
 
-output "ygg_ip1" {
-  value = grid_deployment.d1.vms[0].ygg_ip
+output "planetary_ip1" {
+  value = grid_deployment.d1.vms[0].planetary_ip
 }
-output "ygg_ip2" {
-  value = grid_deployment.d2.vms[0].ygg_ip
+output "planetary_ip2" {
+  value = grid_deployment.d2.vms[0].planetary_ip
 }
 
 output "ipv4_vm1" {

collections/system_administrators/terraform/advanced/terraform_nextcloud_redundant.md

@@ -261,11 +261,11 @@ output "node1_zmachine2_ip" {
   value = grid_deployment.d2.vms[0].ip
 }
 
-output "ygg_ip1" {
-  value = grid_deployment.d1.vms[0].ygg_ip
+output "planetary_ip1" {
+  value = grid_deployment.d1.vms[0].planetary_ip
 }
-output "ygg_ip2" {
-  value = grid_deployment.d2.vms[0].ygg_ip
+output "planetary_ip2" {
+  value = grid_deployment.d2.vms[0].planetary_ip
 }
 
 output "ipv4_vm1" {
@@ -808,11 +808,11 @@ output "node1_zmachine1_ip" {
 #  value = grid_deployment.d2.vms[0].ip
 #}
 
-output "ygg_ip1" {
-  value = grid_deployment.d1.vms[0].ygg_ip
+output "planetary_ip1" {
+  value = grid_deployment.d1.vms[0].planetary_ip
 }
-#output "ygg_ip2" {
-#  value = grid_deployment.d2.vms[0].ygg_ip
+#output "planetary_ip2" {
+#  value = grid_deployment.d2.vms[0].planetary_ip
 #}
 
 output "ipv4_vm1" {

collections/system_administrators/terraform/advanced/terraform_nextcloud_single.md

@@ -213,8 +213,8 @@ output "node1_zmachine1_ip" {
   value = grid_deployment.d1.vms[0].ip
 }
 
-output "ygg_ip1" {
-  value = grid_deployment.d1.vms[0].ygg_ip
+output "planetary_ip1" {
+  value = grid_deployment.d1.vms[0].planetary_ip
 }
 
 output "ipv4_vm1" {

collections/system_administrators/terraform/advanced/terraform_nomad.md

@@ -234,19 +234,19 @@ output "client2_wg_ip" {
 }
 
 output "server1_planetary_ip" {
-  value = grid_deployment.d1.vms[0].ygg_ip
+  value = grid_deployment.d1.vms[0].planetary_ip
 }
 output "server2_planetary_ip" {
-  value = grid_deployment.d1.vms[1].ygg_ip
+  value = grid_deployment.d1.vms[1].planetary_ip
 }
 output "server3_planetary_ip" {
-  value = grid_deployment.d1.vms[2].ygg_ip
+  value = grid_deployment.d1.vms[2].planetary_ip
 }
 output "client1_planetary_ip" {
-  value = grid_deployment.d1.vms[3].ygg_ip
+  value = grid_deployment.d1.vms[3].planetary_ip
 }
 output "client2_planetary_ip" {
-  value = grid_deployment.d1.vms[4].ygg_ip
+  value = grid_deployment.d1.vms[4].planetary_ip
 }
 ```
 

collections/system_administrators/terraform/advanced/terraform_provisioners.md

@@ -61,7 +61,7 @@ resource "grid_deployment" "d1" {
     type  = "ssh"
     user  = "root"
     agent = true
-    host  = grid_deployment.d1.vms[0].ygg_ip
+    host  = grid_deployment.d1.vms[0].planetary_ip
   }
 
   provisioner "remote-exec" {
@@ -88,7 +88,7 @@ resource "grid_deployment" "d1" {
     type     = "ssh"
     user     = "root"
     agent    = true
-    host     = grid_deployment.d1.vms[0].ygg_ip
+    host     = grid_deployment.d1.vms[0].planetary_ip
   }
 ```
 

collections/system_administrators/terraform/advanced/terraform_wireguard_vpn.md

@@ -209,11 +209,11 @@ output "node1_zmachine2_ip" {
   value = grid_deployment.d2.vms[0].ip
 }
 
-output "ygg_ip1" {
-  value = grid_deployment.d1.vms[0].ygg_ip
+output "planetary_ip1" {
+  value = grid_deployment.d1.vms[0].planetary_ip
 }
-output "ygg_ip2" {
-  value = grid_deployment.d2.vms[0].ygg_ip
+output "planetary_ip2" {
+  value = grid_deployment.d2.vms[0].planetary_ip
 }
 
 output "ipv4_vm1" {

collections/system_administrators/terraform/resources/terraform_caprover.md

@@ -136,8 +136,8 @@ resource "grid_deployment" "d0" {
 output "wg_config" {
     value = grid_network.net0.access_wg_config
 }
-output "ygg_ip" {
-    value = grid_deployment.d0.vms[0].ygg_ip
+output "planetary_ip" {
+    value = grid_deployment.d0.vms[0].planetary_ip
 }
 output "vm_ip" {
     value = grid_deployment.d0.vms[0].ip
@@ -427,8 +427,8 @@ resource "grid_deployment" "d2" {
 output "wg_config" {
     value = grid_network.net2.access_wg_config
 }
-output "ygg_ip" {
-    value = grid_deployment.d2.vms[0].ygg_ip
+output "planetary_ip" {
+    value = grid_deployment.d2.vms[0].planetary_ip
 }
 output "vm_ip" {
     value = grid_deployment.d2.vms[0].ip

collections/system_administrators/terraform/resources/terraform_qsfs_on_full_vm.md

@@ -181,8 +181,8 @@ resource "grid_deployment" "qsfs" {
 output "metrics" {
   value = grid_deployment.qsfs.qsfs[0].metrics_endpoint
 }
-output "ygg_ip" {
-  value = grid_deployment.qsfs.vms[0].ygg_ip
+output "planetary_ip" {
+  value = grid_deployment.qsfs.vms[0].planetary_ip
 }
 ```
 

collections/system_administrators/terraform/resources/terraform_qsfs_on_microvm.md

@@ -266,8 +266,8 @@ resource "grid_deployment" "qsfs" {
 output "metrics" {
   value = grid_deployment.qsfs.qsfs[0].metrics_endpoint
 }
-output "ygg_ip" {
-  value = grid_deployment.qsfs.vms[0].ygg_ip
+output "planetary_ip" {
+  value = grid_deployment.qsfs.vms[0].planetary_ip
 }
 ```
 
@@ -337,7 +337,7 @@ To SSH with Planetary Network, write the following:
 ssh root@planetary_IP
 ```
 
-Note that the IP address should be the value of the parameter **ygg_ip** from the Terraform Outputs.
+Note that the IP address should be the value of the parameter **planetary_ip** from the Terraform Outputs.
 
 You now have an SSH connection access to the VM over Planetary Network.
 

collections/system_administrators/terraform/resources/terraform_scheduler.md

@@ -94,15 +94,15 @@ resource "grid_deployment" "d1" {
 output "vm1_ip" {
   value = grid_deployment.d1.vms[0].ip
 }
-output "vm1_ygg_ip" {
-  value = grid_deployment.d1.vms[0].ygg_ip
+output "vm1_planetary_ip" {
+  value = grid_deployment.d1.vms[0].planetary_ip
 }
 
 output "vm2_ip" {
   value = grid_deployment.d1.vms[1].ip
 }
-output "vm2_ygg_ip" {
-  value = grid_deployment.d1.vms[1].ygg_ip
+output "vm2_planetary_ip" {
+  value = grid_deployment.d1.vms[1].planetary_ip
 }
 
 ```

collections/system_administrators/terraform/resources/terraform_vm.md

@@ -87,15 +87,15 @@ resource "grid_deployment" "d1" {
 output "vm1_ip" {
   value = grid_deployment.d1.vms[0].ip
 }
-output "vm1_ygg_ip" {
-  value = grid_deployment.d1.vms[0].ygg_ip
+output "vm1_planetary_ip" {
+  value = grid_deployment.d1.vms[0].planetary_ip
 }
 
 output "vm2_ip" {
   value = grid_deployment.d1.vms[1].ip
 }
-output "vm2_ygg_ip" {
-  value = grid_deployment.d1.vms[1].ygg_ip
+output "vm2_planetary_ip" {
+  value = grid_deployment.d1.vms[1].planetary_ip
 }
 
 ```
@@ -276,7 +276,7 @@ output "public_ip" {
     value = grid_deployment.d1.vms[0].computedip
 }
 
-output "ygg_ip" {
-    value = grid_deployment.d1.vms[0].ygg_ip
+output "planetary_ip" {
+    value = grid_deployment.d1.vms[0].planetary_ip
 }
 ```

collections/system_administrators/terraform/resources/terraform_vm_gateway.md

@@ -77,8 +77,8 @@ output "public_ip" {
     value = split("/",grid_deployment.d1.vms[0].computedip)[0]
 }
 
-output "ygg_ip" {
-    value = grid_deployment.d1.vms[0].ygg_ip
+output "planetary_ip" {
+    value = grid_deployment.d1.vms[0].planetary_ip
 }
 
 ```

collections/system_administrators/terraform/terraform_full_vm.md

@@ -196,8 +196,8 @@ output "node1_zmachine1_ip" {
   value = grid_deployment.d1.vms[0].ip
 }
 
-output "ygg_ip1" {
-  value = grid_deployment.d1.vms[0].ygg_ip
+output "planetary_ip1" {
+  value = grid_deployment.d1.vms[0].planetary_ip
 }
 
 output "ipv4_vm1" {