From b62648cde0e591b4e5007b4c1b29e2d1423d1fe8 Mon Sep 17 00:00:00 2001 From: medina474 Date: Fri, 19 Sep 2025 10:02:52 +0200 Subject: [PATCH] Neo4J --- compose.yaml | 60 ++++++++++++++- data/marque.csv | 2 +- init_metabase.py | 193 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 250 insertions(+), 5 deletions(-) create mode 100644 init_metabase.py diff --git a/compose.yaml b/compose.yaml index 781fb2e..d752996 100644 --- a/compose.yaml +++ b/compose.yaml @@ -17,16 +17,16 @@ services: interval: 10s timeout: 5s retries: 5 - start_period: 10s + start_period: 20s - bi: + superset: image: apache/superset:3.1.3 depends_on: database: condition: service_healthy environment: SUPERSET_CONFIG_PATH: /app/pythonpath/superset_config.py - SUPERSET_SECRET_KEY: ${SUPERSET_SECRET:-!ChangeMe!} + SUPERSET_SECRET_KEY: ${SUPERSET_SECRET:-YOUR_OWN_RANDOM_GENERATED_SECRET_KEY} SUPERSET_LOAD_EXAMPLES: no ports: - "8088:8088" @@ -68,6 +68,15 @@ services: MB_EMAIL_SMTP_PORT: 1025 MB_EMAIL_FROM_ADDRESS: metabase@univ-lorraine.fr + metabase-init: + image: python:3.13-slim + depends_on: + - metabase + working_dir: /app + volumes: + - ./init_metabase.py:/app/init_metabase.py + command: ["python", "init_metabase.py"] + # CouchDB # Single-node document database # https://couchdb.apache.org/ @@ -82,7 +91,8 @@ services: COUCHDB_PASSWORD: ${DB_ROOT_PASSWORD:-!ChangeMe!} mongodb: - image: mongodb/mongodb-community-server:${MONGODB_VERSION:-8.0-ubi8} + image: mongodb/mongodb-community-server:8.0-ubi8 + restart: no volumes: - mongodb_configdb:/data/configdb - mongodb_data:/data/db @@ -92,9 +102,51 @@ services: MONGO_INITDB_ROOT_USERNAME: ${DB_ROOT_USER:-admin} MONGO_INITDB_ROOT_PASSWORD: ${DB_ROOT_PASSWORD:-!ChangeMe!} +# Redis +# In-memory key-value database +# https://redis.io/fr/ + redis: + image: redis:8.2 + restart: no + command: + - 'redis-server' + - '--save 60 1' + - '--loglevel verbose' + - '--requirepass ${DB_ROOT_PASSWORD:-!ChangeMe!}' + ports: + - "6379:6379" + healthcheck: + test: ["CMD", "redis-cli", "-a", "${DB_ROOT_PASSWORD:-!ChangeMe!}", "--raw", "incr", "ping" ] + interval: 30s + timeout: 5s + retries: 3 + start_period: 20s + volumes: + - redis_data:/data + +# Neo4J +# Des graphes ultra-rapides, à l’échelle du pétaoctet +# https://neo4j.com/ + neo4j: + image: neo4j:5.26.12-community + restart: no + ports: + - 7473:7473 + - 7474:7474 + - 7687:7687 + volumes: + - neo4j_data:/data + - neo4j_logs:/logs + environment: + NEO4J_AUTH: neo4j/${DB_ROOT_PASSWORD:-!ChangeMe!} + + volumes: database_data: superset_home: couchdb_data: mongodb_data: mongodb_configdb: + redis_data: + neo4j_data: + neo4j_logs: diff --git a/data/marque.csv b/data/marque.csv index d343e16..c8aeaf5 100644 --- a/data/marque.csv +++ b/data/marque.csv @@ -68,4 +68,4 @@ id,marque 148,Strider 149,Sun Bicycles 151,Surly -151,Trek +152,Trek diff --git a/init_metabase.py b/init_metabase.py new file mode 100644 index 0000000..5887e0a --- /dev/null +++ b/init_metabase.py @@ -0,0 +1,193 @@ +import requests +import time +import sys + +BASE_URL = "http://metabase:3000" + +ADMIN = { + "first_name": "Emmanuel", + "last_name": "Medina", + "email": "emmanuel.medina@univ-lorraine.fr", + "password": "!ChangeMe!" +} + +DB = { + "engine": "postgres", + "name": "ventdest", + "is_on_demand": False, + "is_full_sync": True, + "is_sample": False, + "details": { + "host": "database", + "port": 5432, + "dbname": "sql", + "user": "sql", + "password": "!ChangeMe!", + "ssl": False + } +} + + +def wait_for_metabase(timeout=300): + print("⏳ Attente de Metabase...") + start = time.time() + while time.time() - start < timeout: + try: + r = requests.get(f"{BASE_URL}/api/health", timeout=2) + if r.ok: + print("✅ Metabase est prêt") + return True + except requests.RequestException: + pass + time.sleep(2) + print("❌ Timeout en attendant Metabase") + return False + + +def get_setup_token(): + r = requests.get(f"{BASE_URL}/api/session/properties") + r.raise_for_status() + return r.json().get("setup-token") + + +def run_setup(token): + payload = { + "token": token, + "user": ADMIN, + "prefs": {"site_name": "IUT", "site_locale": "fr"} + } + r = requests.post(f"{BASE_URL}/api/setup", json=payload) + if r.status_code == 400 and "already" in r.text.lower(): + return login(ADMIN["email"], ADMIN["password"]) + r.raise_for_status() + return r.json()["id"] + + +def login(email, password): + r = requests.post(f"{BASE_URL}/api/session", + json={"username": email, "password": password}) + r.raise_for_status() + return r.json()["id"] + + +def database_exists(session_id, name): + headers = {"X-Metabase-Session": session_id} + r = requests.get(f"{BASE_URL}/api/database", headers=headers) + r.raise_for_status() + for db in r.json(): + if db["name"].lower() == name.lower(): + return db["id"] + return None + + +def add_database(session_id, db_config): + db_id = database_exists(session_id, db_config["name"]) + if db_id: + print(f"ℹ️ Base '{db_config['name']}' existe déjà (id={db_id})") + return db_id + headers = {"X-Metabase-Session": session_id} + r = requests.post(f"{BASE_URL}/api/database", + headers=headers, json=db_config) + r.raise_for_status() + db_id = r.json()["id"] + print(f"🗄️ Base '{db_config['name']}' ajoutée (id={db_id})") + return db_id + + +def get_or_create_collection(session_id, name, parent_id=None): + headers = {"X-Metabase-Session": session_id} + r = requests.get(f"{BASE_URL}/api/collection", headers=headers) + r.raise_for_status() + for coll in r.json(): + if coll["name"].lower() == name.lower(): + print(f"ℹ️ Collection '{name}' existe déjà (id={coll['id']})") + return coll["id"] + payload = {"name": name, "color": "#509EE3"} + if parent_id: + payload["parent_id"] = parent_id + r = requests.post(f"{BASE_URL}/api/collection", + headers=headers, json=payload) + r.raise_for_status() + print(f"📁 Collection '{name}' créée (id={r.json()['id']})") + return r.json()["id"] + + +def get_or_create_question(session_id, name, db_id, sql, collection_id): + headers = {"X-Metabase-Session": session_id} + r = requests.get(f"{BASE_URL}/api/card", headers=headers) + r.raise_for_status() + for q in r.json(): + if q["name"].lower() == name.lower(): + print(f"ℹ️ Question '{name}' existe déjà (id={q['id']})") + return q["id"] + + payload = { + "name": name, + "dataset_query": { + "database": db_id, + "type": "native", + "native": {"query": sql} + }, + "display": "table", + "collection_id": collection_id + } + r = requests.post(f"{BASE_URL}/api/card", + headers=headers, json=payload) + r.raise_for_status() + print(f"❓ Question '{name}' créée (id={r.json()['id']})") + return r.json()["id"] + + +def get_or_create_dashboard(session_id, name, collection_id): + headers = {"X-Metabase-Session": session_id} + r = requests.get(f"{BASE_URL}/api/dashboard", headers=headers) + r.raise_for_status() + for d in r.json(): + if d["name"].lower() == name.lower(): + print(f"ℹ️ Dashboard '{name}' existe déjà (id={d['id']})") + return d["id"] + + payload = {"name": name, "collection_id": collection_id} + r = requests.post(f"{BASE_URL}/api/dashboard", + headers=headers, json=payload) + r.raise_for_status() + print(f"📊 Dashboard '{name}' créé (id={r.json()['id']})") + return r.json()["id"] + + +def add_question_to_dashboard(session_id, dashboard_id, card_id): + headers = {"X-Metabase-Session": session_id} + payload = {"cardId": card_id} + r = requests.post(f"{BASE_URL}/api/dashboard/{dashboard_id}/cards", + headers=headers, json=payload) + if r.status_code == 400 and "already" in r.text.lower(): + print(f"ℹ️ Question {card_id} déjà liée au dashboard {dashboard_id}") + return + r.raise_for_status() + print(f"➕ Question {card_id} ajoutée au dashboard {dashboard_id}") + + +if __name__ == "__main__": + if not wait_for_metabase(): + sys.exit(1) + + token = get_setup_token() + session = run_setup(token) + + db_id = add_database(session, DB) + coll_id = get_or_create_collection(session, "IUT Dashboard") + + # Exemple : une question SQL + q1 = get_or_create_question( + session, + "Nombre de lignes", + db_id, + "SELECT COUNT(*) AS total FROM information_schema.tables;", + coll_id + ) + + # Exemple : un dashboard + dash_id = get_or_create_dashboard(session, "Vue d'ensemble", coll_id) + add_question_to_dashboard(session, dash_id, q1) + + print("🎉 Initialisation Metabase terminée avec dashboards/questions")