Feature/opentelemetry

.github/workflows/django_checks.yml

@@ -62,6 +62,7 @@ jobs:
         MVENTORY_DB_USER: ${{ matrix.db_user }}
         MVENTORY_DB_NAME: ${{ matrix.db }}
         MVENTORY_DB_HOST: 127.0.0.1
+        DJANGO_SETTINGS_MODULE: mventory.settings
         MVENTORY_DB_PORT: ${{ matrix.db_port }}
         MVENTORY_DB_PASSWORD: hunter2
         MVENTORY_SECRET_KEY: ${{ secrets.MVENTORY_SECRET_KEY }}

inventory/serializers.py

@@ -1,19 +1,24 @@
 import os
+import logging
 from .utils import OctopartClient
 from .models import Building, Room, StorageUnit, StorageBin, Component, ComponentMeasurementUnit
 from rest_framework import serializers
 
+# Get an instance of a logger
+logger = logging.getLogger(__name__)
 
 class BuildingSerializer(serializers.HyperlinkedModelSerializer):
     class Meta:
         model = Building
         fields = ['url', 'name', 'address', 'postcode']
+        logger.info("Serialising Buildings")
 
 
 class RoomSerializer(serializers.HyperlinkedModelSerializer):
     class Meta:
         model = Room
         fields = ['url', 'name', 'building']
+        logger.info("Serialising Rooms")
 
 class StorageUnitSerializer(serializers.HyperlinkedModelSerializer):
     class Meta:
@@ -32,6 +37,7 @@ def get_octopart_data(self, obj):
         op_data = {}
         if os.getenv("MVENTORY_OCTOPART_API_KEY"):
             if obj.mpn is not None:
+                logger.debug(f"Found Octopart API Key and MPN, retrieving data for {obj.name}")
                 oc = OctopartClient()
                 parts_res = oc.match_mpns([obj.mpn])
                 if parts_res != {}:

inventory/views.py

@@ -1,3 +1,4 @@
+import logging
 from django.shortcuts import render
 
 # Create your views here.
@@ -12,6 +13,9 @@
         ComponentMeasurementUnitSerializer
         )
 
+# Get an instance of a logger
+logger = logging.getLogger(__name__)
+
 
 def index(request):
     return render(request, 'index.html')
@@ -20,6 +24,7 @@ class BuildingViewSet(viewsets.ModelViewSet):
     """
     API endpoint that allows users to be viewed or edited.
     """
+    logger.info("Creating the building view")
     queryset = Building.objects.all().order_by('name')
     search_fields = ['name', 'address', 'postcode']
     filter_backends = (filters.SearchFilter,)

manage.py

@@ -2,11 +2,42 @@
 """Django's command-line utility for administrative tasks."""
 import os
 import sys
+import socket
+from opentelemetry.instrumentation.wsgi import OpenTelemetryMiddleware
+from opentelemetry.instrumentation.django import DjangoInstrumentor
+from opentelemetry.instrumentation.logging import LoggingInstrumentor
+from opentelemetry.instrumentation.mysql import MySQLInstrumentor
+from opentelemetry.instrumentation.requests import RequestsInstrumentor
+from opentelemetry import trace
+from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
+from opentelemetry.sdk.resources import Resource
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import (
+    BatchSpanProcessor,
+    ConsoleSpanExporter,
+    )
+resource = Resource(attributes={
+    "service.name": "mventory",
+    "environment": "dev",
+    "host": socket.gethostname()
+})
 
+trace.set_tracer_provider(TracerProvider(resource=resource))
+tracer = trace.get_tracer(__name__)
+
+otlp_exporter = OTLPSpanExporter(endpoint=f"{os.getenv('OTEL_EXPORTER_OTLP_ENDPOINT')}/v1/traces")
+
+span_processor = BatchSpanProcessor(otlp_exporter)
+
+trace.get_tracer_provider().add_span_processor(span_processor)
+LoggingInstrumentor().instrument(set_logging_format=True)
+RequestsInstrumentor().instrument()
 
 def main():
-    """Run administrative tasks."""
+    DjangoInstrumentor().instrument()
     os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'mventory.settings')
+    os.environ.setdefault("OPENTELEMETRY_PYTHON_DJANGO_INSTRUMENT", "True")
+
     try:
         from django.core.management import execute_from_command_line
     except ImportError as exc:

mventory/settings.py

@@ -12,6 +12,8 @@
 
 import os
 
+import logging_loki
+
 from pathlib import Path
 
 # Build paths inside the project like this: BASE_DIR / 'subdir'.
@@ -140,6 +142,7 @@
 # https://docs.djangoproject.com/en/3.2/howto/static-files/
 
 STATIC_URL = '/static/'
+STATIC_ROOT = os.getenv("MVENTORY_STATIC_ROOT")
 
 # Default primary key field type
 # https://docs.djangoproject.com/en/3.2/ref/settings/#default-auto-field
@@ -154,3 +157,58 @@
 # Octopart
 OCTOPART_API_KEY=os.getenv('MVENTORY_OCTOPART_API_KEY')
 OCTOPART_API_ENDPOINT='https://octopart.com/api/v4/endpoint'
+
+# Logging
+LOGGING = {
+    'version': 1,
+    'disable_existing_loggers': True,
+    'formatters': {
+        'json': {'()': 'pythonjsonlogger.jsonlogger.JsonFormatter'},
+        'standard': {
+            'format': '%(asctime)s %(levelname)s [%(name)s] [%(filename)s:%(lineno)d] [traceID=%(otelTraceID)s spanID=%(otelSpanID)s resource.service.name=%(otelServiceName)s] - %(message)s',
+            'datefmt': '%d-%m-%Y %H:%M:%S'
+        },
+    },
+    'handlers': {
+        'console': {
+            'level': 'INFO',
+            'class': 'logging.StreamHandler',
+            'formatter': 'standard',
+        },
+        'loki': {
+            'level': 'DEBUG',
+            'class': 'logging_loki.LokiHandler',
+            'url': f"{os.getenv('MVENTORY_LOKI_HOST')}/loki/api/v1/push",
+            'tags': {"app": "mventory", "env": "dev"},
+            'version': "1",
+            'formatter': 'json',
+        },
+    },
+    'loggers': {
+        'root': {
+            'handlers': ['loki', 'console'],
+            'level': 'DEBUG',
+            'propagate': True,
+        },
+        'django': {
+            'handlers': ['loki', 'console'],
+            'level': 'DEBUG',
+            'propagate': True,
+        },
+        'django.request': {
+            'handlers': ['loki', 'console'],
+            'level': 'DEBUG',
+            'propagate': True,
+        },
+        'django.server': {
+            'handlers': ['loki', 'console'],
+            'level': 'DEBUG',
+            'propagate': True,
+        },
+        'django.security': {
+            'handlers': ['loki', 'console'],
+            'level': 'DEBUG',
+            'propagate': True,
+        }
+    }
+}

mventory/tracing.py

@@ -0,0 +1,34 @@
+from logging import getLogger
+from pkg_resources import iter_entry_points
+
+from opentelemetry import trace
+from opentelemetry.sdk.resources import Resource
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import (
+    ConsoleSpanExporter,
+)
+
+logger = getLogger(__file__)
+
+initialized = False
+
+def init_tracing():
+  global initialized
+  if initialized:
+    return
+  initialized = True
+
+  provider = TracerProvider()
+  trace.set_tracer_provider(provider)
+
+  provider.add_span_processor(BatchExportSpanProcessor(ConsoleSpanExporter()))
+  auto_instrument()
+
+
+# function should be provided out of box as `opentelemetry.instrumentation.auto_instrumentation.auto_instrument`
+def auto_instrument():
+  for entry_point in iter_entry_points("opentelemetry_instrumentor"):
+      try:
+          entry_point.load()().instrument()
+      except Exception: 
+          logger.exception("Instrumenting of %s failed", entry_point.name)

mventory/wsgi.py

@@ -8,9 +8,49 @@
 """
 
 import os
+import socket
+
+from opentelemetry.instrumentation.wsgi import OpenTelemetryMiddleware
+from opentelemetry.instrumentation.django import DjangoInstrumentor
+from opentelemetry.instrumentation.logging import LoggingInstrumentor
+from opentelemetry.instrumentation.mysql import MySQLInstrumentor
+from opentelemetry import trace
+from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
+from opentelemetry.sdk.resources import Resource
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import (
+    BatchSpanProcessor,
+    ConsoleSpanExporter,
+    )
+resource = Resource(attributes={
+    "service.name": "mventory",
+    "environment": "dev",
+    "host": socket.gethostname()
+})
+
+trace.set_tracer_provider(TracerProvider(resource=resource))
+tracer = trace.get_tracer(__name__)
+
+otlp_exporter = OTLPSpanExporter(endpoint=f"{os.getenv('OTEL_EXPORTER_OTLP_ENDPOINT')}/v1/traces")
+
+span_processor = BatchSpanProcessor(otlp_exporter)
+
+trace.get_tracer_provider().add_span_processor(span_processor)
+
+def request_hook(span, request):
+    print(f"Request: {span.context}")
+
+def response_hook(span, request, response):
+    print(f"Response: {span.context}")
+
+
 
 from django.core.wsgi import get_wsgi_application
+DjangoInstrumentor().instrument(request_hook=request_hook, response_hook=response_hook)
+LoggingInstrumentor().instrument(set_logging_format=True)
 
 os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'mventory.settings')
 
 application = get_wsgi_application()
+application = OpenTelemetryMiddleware(application)
+

requirements.txt

@@ -1,20 +1,47 @@
-asgiref==3.4.1
-backoff==1.11.1
+asgiref==3.3.4
+backoff==1.10.0
 certifi==2021.5.30
 chardet==4.0.0
 charset-normalizer==2.0.3
 Deprecated==1.2.12
-Django==3.2.5
+Django==3.2.9
 django-bulma==0.8.2
 django-loki==0.1.4
 django-prometheus==2.1.0
 djangorestframework==3.12.4
 googleapis-common-protos==1.53.0
 grpcio==1.38.1
 idna==3.2
+mysql-connector-python==8.0.27
 mysqlclient==2.0.3
+opentelemetry-api==1.7.1
+opentelemetry-exporter-jaeger==1.7.1
+opentelemetry-exporter-jaeger-proto-grpc==1.7.1
+opentelemetry-exporter-jaeger-thrift==1.7.1
+opentelemetry-exporter-otlp==1.7.1
+opentelemetry-exporter-otlp-proto-grpc==1.7.1
+opentelemetry-exporter-otlp-proto-http==1.7.1
+opentelemetry-exporter-zipkin==1.7.1
+opentelemetry-exporter-zipkin-json==1.7.1
+opentelemetry-exporter-zipkin-proto-http==1.7.1
+opentelemetry-instrumentation==0.26b1
+opentelemetry-instrumentation-asgi==0.26b1
+opentelemetry-instrumentation-dbapi==0.26b1
+opentelemetry-instrumentation-django==0.26b1
+opentelemetry-instrumentation-logging==0.26b1
+opentelemetry-instrumentation-mysql==0.26b1
+opentelemetry-instrumentation-requests==0.26b1
+opentelemetry-instrumentation-sqlite3==0.26b1
+opentelemetry-instrumentation-urllib==0.26b1
+opentelemetry-instrumentation-wsgi==0.26b1
+opentelemetry-propagator-b3==1.7.1
+opentelemetry-proto==1.7.1
+opentelemetry-sdk==1.7.1
+opentelemetry-semantic-conventions==0.26b1
+opentelemetry-util-http==0.26b1
 prometheus-client==0.11.0
 protobuf==3.17.3
+python-json-logger==2.0.2
 python-logging-loki==0.3.1
 pytz==2021.1
 requests==2.26.0

uwsgi.ini

@@ -0,0 +1,7 @@
+[uwsgi]
+socket = 127.0.0.1:3031
+wsgi-file = mventory/wsgi.py
+processes = 1
+threads = 1
+stats = :9191
+log-format = %(addr) - %(user) [%(ltime)] "%(method) %(uri) %(proto)" %(status) %(size) "%(referer)" "%(uagent)" %(var.trace_id) %(var.span_id)