diff --git a/web/migrations/0003_lookupdata_date_time.py b/web/migrations/0003_lookupdata_date_time.py new file mode 100644 index 000000000..86a57aad8 --- /dev/null +++ b/web/migrations/0003_lookupdata_date_time.py @@ -0,0 +1,20 @@ +# Generated by Django 4.2.27 on 2025-12-28 00:20 + +from django.db import migrations, models +import django.utils.timezone + + +class Migration(migrations.Migration): + + dependencies = [ + ('web', '0002_lookupdata_version1_lookupdata_version2'), + ] + + operations = [ + migrations.AddField( + model_name='lookupdata', + name='date_time', + field=models.DateTimeField(auto_now_add=True, default=django.utils.timezone.now), + preserve_default=False, + ), + ] diff --git a/web/migrations/0004_missinglookup.py b/web/migrations/0004_missinglookup.py new file mode 100644 index 000000000..2b4a2ea02 --- /dev/null +++ b/web/migrations/0004_missinglookup.py @@ -0,0 +1,25 @@ +# Generated by Django 4.2.27 on 2025-12-28 00:33 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('web', '0003_lookupdata_date_time'), + ] + + operations = [ + migrations.CreateModel( + name='MissingLookup', + fields=[ + ('id', models.BigAutoField(primary_key=True, serialize=False)), + ('date_time', models.DateTimeField(auto_now_add=True)), + ('item_type', models.CharField(max_length=20)), + ('item_value', models.CharField(max_length=100)), + ('language_context', models.CharField(blank=True, max_length=50, null=True)), + ('site_visit', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='web.sitevisit')), + ], + ), + ] diff --git a/web/models.py b/web/models.py index c7c1dea89..0236e79ca 100644 --- a/web/models.py +++ b/web/models.py @@ -377,10 +377,19 @@ class SiteVisit(models.Model): class LookupData(models.Model): id = models.BigAutoField(primary_key=True) - date_time = models.DateTimeField + date_time = models.DateTimeField(auto_now_add=True) language1 = models.CharField(max_length=50) version1 = models.CharField(max_length=20, default='') language2 = models.CharField(max_length=50) version2 = models.CharField(max_length=20, default='') structure = models.CharField(max_length=50) site_visit = models.ForeignKey(SiteVisit, on_delete=models.CASCADE) + + +class MissingLookup(models.Model): + id = models.BigAutoField(primary_key=True) + date_time = models.DateTimeField(auto_now_add=True) + item_type = models.CharField(max_length=20) # 'language', 'structure', 'concept' + item_value = models.CharField(max_length=100) + language_context = models.CharField(max_length=50, blank=True, null=True) + site_visit = models.ForeignKey(SiteVisit, on_delete=models.CASCADE) diff --git a/web/static/js/contributors.js b/web/static/js/contributors.js index 5ca10c861..e56fd9e14 100644 --- a/web/static/js/contributors.js +++ b/web/static/js/contributors.js @@ -26,4 +26,20 @@ document.addEventListener("DOMContentLoaded", function () { contributorsRequest.onerror = function () { document.querySelector("#contributors").innerHTML = "multiple"; }; + + var repoRequest = new XMLHttpRequest(); + repoRequest.open( + "GET", + "https://api.github.com/repos/codethesaurus/codethesaur.us" + ); + repoRequest.send(); + + repoRequest.onload = function () { + if (repoRequest.status === 200) { + let repoData = JSON.parse(repoRequest.responseText); + let lastUpdate = new Date(repoData.pushed_at); + let options = { year: 'numeric', month: 'long', day: 'numeric' }; + document.querySelector("#last-update").innerHTML = lastUpdate.toLocaleDateString(undefined, options); + } + }; }); diff --git a/web/templates/base.html b/web/templates/base.html index 6a74a1d60..7949c57ac 100644 --- a/web/templates/base.html +++ b/web/templates/base.html @@ -55,6 +55,9 @@
  • About
    • +
  • + Statistics +
  • Docs
    • @@ -79,6 +82,7 @@

    Made with ❤ by Sarah Withee and contributors.

    +

    Last GitHub update:

    Want to help out? Check the project out on GitHub.

    diff --git a/web/templates/statistics.html b/web/templates/statistics.html new file mode 100644 index 000000000..1afb0a263 --- /dev/null +++ b/web/templates/statistics.html @@ -0,0 +1,273 @@ +{% extends 'base.html' %} + +{% block content %} +
    +
    +
    +

    Site Statistics

    +

    Insights into how developers are using Code Thesaurus.

    +
    +
    + +
    +
    +
    +
    +
    Overall Activity
    +
    +
    +
    +
    +

    {{ total_visits }}

    +

    Total Visits

    +
    +
    +

    {{ total_lookups }}

    +

    Total Lookups

    +
    +
    +
    +
    +
    +

    {{ unique_comparisons_count }}

    + Unique Lang Comparisons +
    +
    +

    {{ unique_structures_count }}

    + Unique Category Lookups +
    +
    +
    +
    +
    +
    +
    +
    +
    Language Popularity
    +
    +
    + +
    +
    +
    +
    + +
    +
    +
    +
    +
    Most Popular Specific Concept Lookups
    +
    +
    +

    Comparing how often specific concepts are looked up for each language (e.g., "Javascript functions").

    + +
    +
    +
    +
    + +
    +
    +
    +
    +
    Recent Lookups
    +
    +
    + {% if recent_lookups %} + + + + + + + + + + + {% for lookup in recent_lookups %} + + + + + + + {% endfor %} + +
    TimeLanguage 1Language 2Concept
    {{ lookup.date_time|date:"Y-m-d H:i" }}{{ lookup.lang1 }}{% if lookup.lang2 %}{{ lookup.lang2 }}{% else %}-{% endif %}{{ lookup.structure }}
    + {% else %} +

    No recent lookups recorded.

    + {% endif %} +
    +
    +
    +
    + +
    +
    +
    +
    +
    Top Comparisons
    +
    +
    + {% if popular_comparisons %} + + + + + + + + + + {% for comp in popular_comparisons %} + + + + + + {% endfor %} + +
    Language 1Language 2Count
    {{ comp.lang1 }}{{ comp.lang2 }}{{ comp.count }}
    + {% else %} +

    No comparisons made yet.

    + {% endif %} +
    +
    +
    +
    + +
    +
    +
    +
    +
    Most Requested Missing Items
    +
    +
    +

    Tracking languages, structures, or concepts that were requested but are currently missing from the site.

    + {% if missing_items %} + + + + + + + + + {% for item in missing_items %} + + + + + {% endfor %} + +
    ItemRequests
    + {% if item.type == 'language' %} + Language + {% elif item.type == 'structure' %} + Structure + {% else %} + Concept + {% endif %} + {{ item.label }} + {{ item.count }}
    + {% else %} +

    No missing items requested yet.

    + {% endif %} +
    +
    +
    +
    + +
    +
    +
    +
    +
    Category Popularity
    +
    +
    + +
    +
    +
    +
    +
    + + + +{% endblock %} diff --git a/web/tests/test_views.py b/web/tests/test_views.py index 2474ef985..39c810b12 100644 --- a/web/tests/test_views.py +++ b/web/tests/test_views.py @@ -1,4 +1,5 @@ """Tests for the views of codethesaur.us""" +import logging from http import HTTPStatus from django.test import TestCase @@ -6,6 +7,14 @@ from web.models import LookupData +def setUpModule(): + logging.disable(logging.CRITICAL) + + +def tearDownModule(): + logging.disable(logging.NOTSET) + + class TestViews(TestCase): """TestCase for the views""" diff --git a/web/urls.py b/web/urls.py index 35ac3d265..61b67a3c8 100644 --- a/web/urls.py +++ b/web/urls.py @@ -16,6 +16,9 @@ # /about/ path('about/', views.about, name='about'), + # /statistics/ + path('statistics/', views.statistics, name='statistics'), + # /compare/lang1/lang2 #path('//', views.detail, name='detail') # /compare/ diff --git a/web/views.py b/web/views.py index 062f0f033..39100fab0 100644 --- a/web/views.py +++ b/web/views.py @@ -10,6 +10,8 @@ HttpResponseNotFound, HttpResponseServerError ) +from django.db import transaction +from django.db.models import Count, Q from django.shortcuts import HttpResponse, render from django.utils.html import escape, strip_tags from django.views.decorators.http import require_http_methods @@ -24,6 +26,7 @@ LookupData, MetaInfo, MissingLanguageError, + MissingLookup, MissingStructureError, SiteVisit, ) @@ -31,35 +34,62 @@ def store_url_info(request): - if 'HTTP_USER_AGENT' in request.META: - user_agent = request.META['HTTP_USER_AGENT'] - else: - user_agent = "" + try: + if 'HTTP_USER_AGENT' in request.META: + user_agent = request.META['HTTP_USER_AGENT'] + else: + user_agent = "" + + if 'HTTP_REFERER' in request.META: + referer = request.META['HTTP_REFERER'] + else: + referer = "" + + visit = SiteVisit( + url=request.get_full_path(), + user_agent=user_agent, + referer=referer, + ) + with transaction.atomic(): + visit.save() + return visit + except Exception as e: + logging.error(f"Failed to store URL info: {e}") + return None - if 'HTTP_REFERER' in request.META: - referer = request.META['HTTP_REFERER'] - else: - referer = "" - visit = SiteVisit( - url=request.get_full_path(), - user_agent=user_agent, - referer=referer, - ) - visit.save() - return visit +def store_lookup_info(request, visit, language1, version1, language2, version2, structure): + if not visit: + return + try: + info = LookupData( + language1=language1, + version1=version1, + language2=language2, + version2=version2, + structure=structure, + site_visit=visit + ) + with transaction.atomic(): + info.save() + except Exception as e: + logging.error(f"Failed to store lookup info: {e}") -def store_lookup_info(request, visit, language1, version1, language2, version2, structure): - info = LookupData( - language1=language1, - version1=version1, - language2=language2, - version2=version2, - structure=structure, - site_visit=visit - ) - info.save() +def store_missing_info(visit, item_type, item_value, language_context=None): + if not visit: + return + try: + info = MissingLookup( + item_type=item_type, + item_value=item_value, + language_context=language_context, + site_visit=visit + ) + with transaction.atomic(): + info.save() + except Exception as e: + logging.error(f"Failed to store missing info: {e}") @require_http_methods(['GET']) @@ -114,6 +144,179 @@ def index(request): return render(request, 'index.html', content) +@require_http_methods(['GET']) +def statistics(request): + """ + Renders the statistics page (/statistics/) + + :param request: HttpRequest object + :return: HttpResponse object with rendered object of the page + """ + store_url_info(request) + + meta_info = MetaInfo() + + # Most popular languages (considering both language1 and language2) + # We need to aggregate counts for each language across both fields. + # A simple way is to get counts for each and then merge them in Python. + lang1_counts = LookupData.objects.values('language1').annotate(count=Count('language1')) + lang2_counts = LookupData.objects.exclude(language2='').values('language2').annotate(count=Count('language2')) + + combined_counts = {} + for item in lang1_counts: + lang = item['language1'] + combined_counts[lang] = combined_counts.get(lang, 0) + item['count'] + for item in lang2_counts: + lang = item['language2'] + combined_counts[lang] = combined_counts.get(lang, 0) + item['count'] + + sorted_langs = sorted(combined_counts.items(), key=lambda x: x[1], reverse=True) + popular_languages = [] + for lang_key, count in sorted_langs[:10]: + try: + name = meta_info.language_name(lang_key) + except (KeyError, MissingLanguageError): + name = lang_key + popular_languages.append({'name': name, 'count': count}) + + # Most popular structures + structure_counts = LookupData.objects.values('structure').annotate(count=Count('structure')).order_by('-count')[:10] + popular_structures = [] + for item in structure_counts: + try: + name = meta_info.structure_name(item['structure']) + except (KeyError, MissingStructureError): + name = item['structure'] + popular_structures.append({'name': name, 'count': item['count']}) + + # Most popular comparisons + # Using a technique to ensure (lang1, lang2) is treated the same as (lang2, lang1) if we wanted to, + # but let's keep it simple and just look at pairs as they are. + comparison_counts = LookupData.objects.exclude(language2='').values('language1', 'language2').annotate(count=Count('id')).order_by('-count')[:10] + popular_comparisons = [] + for item in comparison_counts: + try: + name1 = meta_info.language_name(item['language1']) + except (KeyError, MissingLanguageError): + name1 = item['language1'] + try: + name2 = meta_info.language_name(item['language2']) + except (KeyError, MissingLanguageError): + name2 = item['language2'] + popular_comparisons.append({'lang1': name1, 'lang2': name2, 'count': item['count']}) + + total_visits = SiteVisit.objects.count() + total_lookups = LookupData.objects.count() + + # Unique language comparisons + unique_comparisons_count = LookupData.objects.exclude(language2='').values('language1', 'language2').distinct().count() + + # Unique concept categories (structures) looked up + unique_structures_count = LookupData.objects.values('structure').distinct().count() + + # Most popular concept-language pairs (e.g., Javascript functions) + concept_lang_counts = {} + for entry in LookupData.objects.all(): + # Count for language 1 + key1 = (entry.language1, entry.structure) + concept_lang_counts[key1] = concept_lang_counts.get(key1, 0) + 1 + # Count for language 2 if it exists + if entry.language2: + key2 = (entry.language2, entry.structure) + concept_lang_counts[key2] = concept_lang_counts.get(key2, 0) + 1 + + sorted_concept_langs = sorted(concept_lang_counts.items(), key=lambda x: x[1], reverse=True) + popular_concept_langs = [] + for (lang_key, struct_key), count in sorted_concept_langs[:10]: + try: + lang_name = meta_info.language_name(lang_key) + except (KeyError, MissingLanguageError): + lang_name = lang_key + try: + struct_name = meta_info.structure_name(struct_key) + except (KeyError, MissingStructureError): + struct_name = struct_key + popular_concept_langs.append({ + 'label': f"{lang_name} {struct_name}", + 'lang': lang_name, + 'struct': struct_name, + 'count': count + }) + + # Recent lookups + recent_lookups_query = LookupData.objects.order_by('-date_time')[:10] + recent_lookups = [] + for item in recent_lookups_query: + try: + name1 = meta_info.language_name(item.language1) + except (KeyError, MissingLanguageError): + name1 = item.language1 + try: + name2 = meta_info.language_name(item.language2) if item.language2 else None + except (KeyError, MissingLanguageError): + name2 = item.language2 + + try: + struct_name = meta_info.structure_name(item.structure) + except (KeyError, MissingStructureError): + struct_name = item.structure + + recent_lookups.append({ + 'lang1': name1, + 'lang2': name2, + 'structure': struct_name, + 'date_time': item.date_time + }) + + # Missing items statistics + missing_items_counts = MissingLookup.objects.values('item_type', 'item_value', 'language_context') \ + .annotate(count=Count('id')).order_by('-count')[:15] + + missing_items = [] + for item in missing_items_counts: + label = item['item_value'] + if item['item_type'] == 'language': + label = f"Language: {item['item_value']}" + elif item['item_type'] == 'structure': + try: + lang_name = meta_info.language_name(item['language_context']) + except (KeyError, MissingLanguageError): + lang_name = item['language_context'] + label = f"Structure: {item['item_value']} (for {lang_name})" + elif item['item_type'] == 'concept': + try: + lang_name = meta_info.language_name(item['language_context']) + except (KeyError, MissingLanguageError): + lang_name = item['language_context'] + label = f"Concept: {item['item_value']} (missing in {lang_name})" + + missing_items.append({ + 'label': label, + 'count': item['count'], + 'type': item['item_type'] + }) + + import json + context = { + 'title': 'Statistics', + 'popular_languages': popular_languages, + 'popular_structures': popular_structures, + 'popular_comparisons': popular_comparisons, + 'popular_concept_langs': popular_concept_langs, + 'recent_lookups': recent_lookups, + 'missing_items': missing_items, + 'total_visits': total_visits, + 'total_lookups': total_lookups, + 'unique_comparisons_count': unique_comparisons_count, + 'unique_structures_count': unique_structures_count, + 'popular_languages_json': json.dumps(popular_languages), + 'popular_structures_json': json.dumps(popular_structures), + 'popular_concept_langs_json': json.dumps(popular_concept_langs), + } + + return render(request, 'statistics.html', context) + + @require_http_methods(['GET']) def about(request): """ @@ -155,6 +358,12 @@ def concepts(request): try: languages = meta_info.load_languages(language_strings, meta_structure) except MissingStructureError as missing_structure: + store_missing_info( + visit, + 'structure', + missing_structure.structure.key, + missing_structure.language_key + ) return HttpResponseNotFound(render( request, "error_missing_structure.html", @@ -172,6 +381,7 @@ def concepts(request): } )) except MissingLanguageError as missing_language: + store_missing_info(visit, 'language', missing_language.key) errors.append(f"The language \"{missing_language.key}\" isn't valid. \ Double-check your URL and try again.") @@ -193,7 +403,7 @@ def concepts(request): for (category_key, category) in meta_structure.categories.items(): concept_keys = list(category.keys()) - concepts_list = [concepts_data(key, name, languages, lexers) for (key, name) in category.items()] + concepts_list = [concepts_data(key, name, languages, lexers, visit) for (key, name) in category.items()] category_entry = { "key": category_key, @@ -304,9 +514,12 @@ def error_handler_500_server_error(request): :param request: HttpRequest object :return: HttpResponse object with rendered object of the page """ - store_url_info(request) + try: + store_url_info(request) + except Exception: + pass - logging.error(request) + logging.error(f"500 error at {request.get_full_path()}") response = render(request, 'error500.html') return HttpResponseServerError(response) @@ -358,7 +571,7 @@ def format_comment_for_display(concept_key, lang): return lang.concept_comment(concept_key) -def concepts_data(key, name, languages, lexers=None): +def concepts_data(key, name, languages, lexers=None, visit=None): """ Generates the comparision object of a single concept @@ -366,11 +579,17 @@ def concepts_data(key, name, languages, lexers=None): :param name: name of the concept :param languages: list of languages to compare / get a reference for :param lexers: optional list of pre-fetched lexers corresponding to languages + :param visit: optional SiteVisit for logging missing items :return: string with code with applied HTML formatting """ data = [] for i, lang in enumerate(languages): lexer = lexers[i] if lexers else None + + # Log if concept is not implemented + if visit and not lang.concept_implemented(key): + store_missing_info(visit, 'concept', key, lang.key) + data.append({ "code": format_code_for_display(key, lang, lexer), "comment": format_comment_for_display(key, lang) @@ -439,9 +658,16 @@ def api_reference(request, structure_key, lang, version): try: response = lang_obj.load_filled_concepts(structure_key, version) except Exception as e: + # Determine if it's a language or structure issue + # If Language(lang, "") failed to find versions, it might be a language issue + if not lang_obj.versions(): + store_missing_info(visit, 'language', lang) + else: + store_missing_info(visit, 'structure', structure_key, lang) return error_handler_404_not_found(request, e) if response is False: + store_missing_info(visit, 'structure', structure_key, lang) return HttpResponseNotFound() store_lookup_info( @@ -470,9 +696,15 @@ def api_compare(request, structure_key, lang1, version1, lang2, version2): """ visit = store_url_info(request) - response = Language(lang1, "").load_comparison(structure_key, lang2, version2, version1) + try: + response = Language(lang1, "").load_comparison(structure_key, lang2, version2, version1) + except Exception: + # Simple logging for now + store_missing_info(visit, 'structure', structure_key, f"{lang1}/{lang2}") + return HttpResponseNotFound() if response is False: + store_missing_info(visit, 'structure', structure_key, f"{lang1}/{lang2}") return HttpResponseNotFound() store_lookup_info(