...

Co-authored-by: Omdanii <mahmmoud.hassanein@gmail.com>

research/openrpc/convert.py

@@ -0,0 +1,16 @@
+
+import json
+from jinja2 import Template
+
+# load OpenRPC spec
+with open("openrpc.json") as f:
+    spec = json.load(f)
+
+# load template
+with open("openrpc_to_md.j2") as f:
+    tmpl = Template(f.read())
+
+# render markdown
+output = tmpl.render(spec=spec)
+
+print(output)

research/openrpc/dense_md_to_openrpc.py

@@ -0,0 +1,163 @@
+import re
+import yaml
+import json
+import requests
+from collections import defaultdict
+from jsonschema import validate, Draft7Validator
+
+# --- Load OpenRPC meta-schema ---
+def load_openrpc_schema():
+    url = "https://open-rpc.github.io/meta-schema.json"
+    resp = requests.get(url)
+    resp.raise_for_status()
+    return resp.json()
+
+OPENRPC_SCHEMA = load_openrpc_schema()
+
+def parse_dense_markdown(md_text: str):
+    methods = []
+    schemas = {}
+
+    # Split into sections
+    sections = re.split(r"^# ", md_text, flags=re.M)
+    secmap = {}
+    for sec in sections:
+        if not sec.strip():
+            continue
+        title, _, body = sec.partition("\n")
+        secmap[title.strip()] = body.strip()
+
+    # --- Methods ---
+    if "Methods" in secmap:
+        method_blocks = re.split(r"^## ", secmap["Methods"], flags=re.M)
+        for block in method_blocks:
+            if not block.strip():
+                continue
+            name, _, body = block.partition("\n")
+            method = {"name": name.strip(), "params": [], "result": {}}
+
+            lines = [l.rstrip() for l in body.splitlines() if l.strip()]
+
+            # description before **Params**
+            desc_lines = []
+            i = 0
+            while i < len(lines) and not lines[i].startswith("**Params**"):
+                desc_lines.append(lines[i])
+                i += 1
+            method["description"] = " ".join(desc_lines).strip()
+
+            # Params
+            if "**Params**" in lines:
+                pi = lines.index("**Params**") + 1
+                while pi < len(lines) and not lines[pi].startswith("**Result**"):
+                    line = lines[pi].lstrip("-").strip()
+                    if line and not line.lower().startswith("none"):
+                        pname, _, pdesc = line.partition("(")
+                        pname = pname.strip()
+                        ptype = pdesc.split(",")[0].replace(")", "").strip() if pdesc else "object"
+                        required = "required" in line
+                        param = {
+                            "name": pname,
+                            "required": required,
+                            "schema": {"type": map_type(ptype)}
+                        }
+                        method["params"].append(param)
+                    pi += 1
+
+            # Result
+            if "**Result**" in lines:
+                ri = lines.index("**Result**") + 1
+                if ri < len(lines):
+                    rline = lines[ri].lstrip("-").strip()
+                    rname, _, rtype = rline.partition(":")
+                    rname = rname.strip()
+                    rtype = rtype.strip()
+                    schema = parse_result_type(rtype)
+                    method["result"] = {"name": rname, "schema": schema}
+
+            methods.append(method)
+
+    # --- Schemas ---
+    if "Schemas" in secmap:
+        schema_blocks = re.split(r"^## ", secmap["Schemas"], flags=re.M)
+        for block in schema_blocks:
+            if not block.strip():
+                continue
+            name, _, body = block.partition("\n")
+            lines = body.splitlines()
+            code_block = []
+            req_fields = []
+            in_yaml = False
+            for l in lines:
+                if l.strip().startswith("```yaml"):
+                    in_yaml = True
+                    continue
+                elif l.strip().startswith("```"):
+                    in_yaml = False
+                    continue
+                if in_yaml:
+                    code_block.append(l)
+                if l.strip().startswith("*Required:"):
+                    req_fields = [x.strip() for x in l.split(":")[1].split(",")]
+            if code_block:
+                schema_def = yaml.safe_load("\n".join(code_block))
+                props = {}
+                for fname, ftype in schema_def.items():
+                    if isinstance(ftype, str):
+                        base = ftype.split("#")[0].strip()
+                        desc = ftype.split("#")[1].strip() if "#" in ftype else None
+                        props[fname] = {"type": map_type(base)}
+                        if desc:
+                            props[fname]["description"] = desc
+                schemas[name.strip()] = {
+                    "type": "object",
+                    "properties": props,
+                    "required": req_fields
+                }
+
+    spec = {"openrpc": "1.0.0", "methods": methods, "components": {"schemas": schemas}}
+
+    # --- Validate against OpenRPC schema ---
+    validator = Draft7Validator(OPENRPC_SCHEMA)
+    errors = sorted(validator.iter_errors(spec), key=lambda e: e.path)
+    if errors:
+        for err in errors:
+            print(f"❌ Validation error at {list(err.path)}: {err.message}")
+        raise ValueError("Generated spec is not valid OpenRPC")
+    else:
+        print("✅ Spec is valid OpenRPC")
+
+    return spec
+
+# --- Helpers ---
+def map_type(t):
+    mapping = {"int": "integer", "str": "string", "bool": "boolean", "object": "object"}
+    return mapping.get(t, t)
+
+def parse_result_type(rtype):
+    if "|" in rtype:
+        oneofs = []
+        for variant in rtype.split("|"):
+            v = variant.strip()
+            if v.startswith("[") and v.endswith("]"):
+                oneofs.append({"type": "array", "items": {"$ref": f"#/components/schemas/{v[1:-1]}"}})
+            else:
+                oneofs.append({"$ref": f"#/components/schemas/{v}"})
+        return {"oneOf": oneofs}
+    elif rtype.startswith("[") and rtype.endswith("]"):
+        return {"type": "array", "items": {"type": map_type(rtype[1:-1])}}
+    elif rtype in ["int", "str", "bool", "object"]:
+        return {"type": map_type(rtype)}
+    else:
+        return {"$ref": f"#/components/schemas/{rtype}"}
+
+
+# --- Example usage ---
+if __name__ == "__main__":
+    with open("dense.md") as f:
+        md = f.read()
+
+    spec = parse_dense_markdown(md)
+
+    with open("openrpc.json", "w") as f:
+        json.dump(spec, f, indent=2)

research/openrpc/install.sh

@@ -0,0 +1 @@
+pip install jsonschema requests

research/openrpc/openrpc2md.md

@@ -0,0 +1,155 @@
+
+
+# Instructions: Converting OpenRPC → Markdown Shorthand
+
+## Purpose
+
+Transform an OpenRPC specification (JSON or YAML) into a **dense Markdown format** that is:
+
+* Human-readable
+* AI-parseable
+* Compact (minimal boilerplate)
+* Faithful to the original semantics
+
+---
+
+## General Rules
+
+1. **Drop metadata**
+   Ignore `info`, `servers`, and other metadata not relevant to API usage.
+
+2. **Two main sections**
+
+   * `# Methods`
+   * `# Schemas`
+
+3. **Method representation**
+   Each method gets its own `## {method_name}` block:
+
+   * First line: *description* (if present).
+   * **Params** section:
+
+     * List each parameter as `- name: TYPE (required?)`
+     * Inline object properties using nested bullet lists or YAML block if complex.
+   * **Result** section:
+
+     * Same style as params.
+     * Use shorthand for references:
+
+       * `$ref: "#/components/schemas/Comment"` → `Comment`
+       * Array of refs → `[Comment]`
+
+4. **Schema representation**
+   Each schema gets its own `## {SchemaName}` block:
+
+   * Use fenced YAML block for properties:
+
+     ```yaml
+     field: TYPE   # description
+     ```
+   * List required fields below the block:
+     `*Required: field1, field2*`
+
+---
+
+## Type Conventions
+
+* `type: integer` → `int`
+* `type: string` → `str`
+* `type: boolean` → `bool`
+* `type: object` → `object`
+* `type: array` → `[TYPE]` (if items defined)
+* `oneOf: [SchemaA, SchemaB]` → `SchemaA | SchemaB`
+
+---
+
+## Handling Complex Types
+
+1. **Objects inside parameters or results**
+
+   * If small (≤3 fields), inline as `{ field1: TYPE, field2: TYPE }`.
+   * If large, expand as YAML block.
+
+   Example:
+
+   ```markdown
+   - args (object, required)
+     - id: int
+     - parent: int
+   ```
+
+2. **Nested schemas (structs in structs)**
+
+   * Inline only the top level, reference nested schemas by name.
+   * If the nested schema is not declared in `components/schemas`, define it under `# Schemas`.
+
+   Example:
+
+   ```markdown
+   ## user_create
+   **Params**
+   - user: UserProfile  (struct defined below)
+   ```
+
+3. **Arrays**
+
+   * `[TYPE]` for primitives (e.g., `[int]`).
+   * `[SchemaName]` for objects.
+
+4. **Results with multiple options**
+
+   * Use `SchemaA | [SchemaA]` for "oneOf".
+
+---
+
+## Example Conversion
+
+OpenRPC (fragment):
+
+```json
+{
+  "name": "comment_get",
+  "description": "Retrieve comments",
+  "params": [
+    {
+      "name": "args",
+      "required": true,
+      "schema": {
+        "type": "object",
+        "properties": {
+          "id": { "type": "integer" },
+          "author": { "type": "integer" }
+        }
+      }
+    }
+  ],
+  "result": {
+    "name": "comments",
+    "schema": {
+      "oneOf": [
+        { "$ref": "#/components/schemas/Comment" },
+        { "type": "array", "items": { "$ref": "#/components/schemas/Comment" } }
+      ]
+    }
+  }
+}
+```
+
+Markdown:
+
+```markdown
+## comment_get
+Retrieve comments
+
+**Params**
+- args (object, required)  
+  - id: int  
+  - author: int  
+
+**Result**
+- comments: Comment | [Comment]
+```
+
+---
+
+This way, any AI (or human) can deterministically map **OpenRPC → Markdown shorthand**.

research/openrpc/roundtrip_test.py

@@ -0,0 +1,53 @@
+import json
+import sys
+from dense_md_to_openrpc import parse_dense_markdown
+from jinja2 import Template
+
+# Load the Jinja2 template for JSON → Markdown
+with open("openrpc_to_md.j2") as f:
+    md_template = Template(f.read())
+
+def roundtrip(md_path: str):
+    print(f"🔄 Round-trip test for {md_path}")
+
+    # 1. Markdown → JSON
+    with open(md_path) as f:
+        md_text = f.read()
+    spec1 = parse_dense_markdown(md_text)
+    print("✅ Step 1: Markdown → JSON")
+
+    # 2. Validate already happens inside parse_dense_markdown
+
+    # 3. JSON → Markdown
+    md2 = md_template.render(spec=spec1)
+    print("✅ Step 2: JSON → Markdown")
+
+    # 4. Markdown → JSON again
+    spec2 = parse_dense_markdown(md2)
+    print("✅ Step 3: Markdown → JSON (again)")
+
+    # 5. Compare results
+    spec1_str = json.dumps(spec1, sort_keys=True, indent=2)
+    spec2_str = json.dumps(spec2, sort_keys=True, indent=2)
+
+    if spec1_str == spec2_str:
+        print("🎉 Round-trip is stable! (JSON1 == JSON2)")
+    else:
+        print("⚠️ Round-trip mismatch!")
+        with open("spec1.json", "w") as f:
+            f.write(spec1_str)
+        with open("spec2.json", "w") as f:
+            f.write(spec2_str)
+        with open("roundtrip_diff.md", "w") as f:
+            f.write("## Original JSON (Markdown → JSON)\n```json\n")
+            f.write(spec1_str + "\n```\n")
+            f.write("\n## After Round-trip (JSON → Markdown → JSON)\n```json\n")
+            f.write(spec2_str + "\n```\n")
+        print("👉 Differences written to spec1.json, spec2.json, and roundtrip_diff.md")
+
+if __name__ == "__main__":
+    if len(sys.argv) < 2:
+        print("Usage: python roundtrip_test.py dense.md")
+        sys.exit(1)
+
+    roundtrip(sys.argv[1])

research/openrpc/schema.j2

@@ -0,0 +1,56 @@
+# Methods
+
+{% for method in spec.methods %}
+## {{ method.name }}
+{{ method.description or "" }}
+
+**Params**
+{% if method.params|length == 0 %}
+- none
+{% else %}
+{% for p in method.params %}
+- {{ p.name }} ({{ p.schema.type or "object" }}{% if p.required %}, required{% endif %})
+{% if p.schema.properties %}
+{% for field, fdef in p.schema.properties.items() %}
+  - {{ field }}: {{ fdef.type or "object" }}{% if fdef.description %} — {{ fdef.description }}{% endif %}
+{% endfor %}
+{% endif %}
+{% if p.schema["$ref"] %}
+  - {{ p.schema["$ref"].split("/")[-1] }}
+{% endif %}
+{% endfor %}
+{% endif %}
+
+**Result**
+- {{ method.result.name }}:
+{% if method.result.schema.oneOf %}
+  {% for variant in method.result.schema.oneOf %}
+    {% if variant["$ref"] %}
+    - {{ variant["$ref"].split("/")[-1] }}
+    {% elif variant.type == "array" %}
+    - [{{ variant.items["$ref"].split("/")[-1] }}]
+    {% else %}
+    - {{ variant.type }}
+    {% endif %}
+  {% endfor %}
+{% elif method.result.schema["$ref"] %}
+  {{ method.result.schema["$ref"].split("/")[-1] }}
+{% elif method.result.schema.type == "array" %}
+  [{{ method.result.schema.items.type or method.result.schema.items["$ref"].split("/")[-1] }}]
+{% elif method.result.schema.type %}
+  {{ method.result.schema.type }}
+{% else %}
+  object
+{% endif %}
+
+---
+{% endfor %}
+
+# Schemas
+
+{% for name, schema in spec.components.schemas.items() %}
+## {{ name }}
+```yaml
+{% for field, fdef in schema.properties.items() %}
+{{ field }}: {{ fdef.type or "object" }}{% if fdef.description %}   # {{ fdef.description }}{% endif %}
+{% endfor %}