diff --git a/.github/workflows/00_smoketest.yaml b/.github/workflows/00_smoketest.yaml
index 63dda185ccd..8cf41080179 100644
--- a/.github/workflows/00_smoketest.yaml
+++ b/.github/workflows/00_smoketest.yaml
@@ -7,18 +7,18 @@ on:
 jobs:
   linux:
     name: Run LMS to see whether it crashes immediately...
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
 
     steps:
       - name: Make sure we're running as non-root
         run: |
           if [ "$RUNNER_USER" == "root" ]; then
             echo Specify non-root image when testing with act:
-            echo act -P ubuntu-22.04=ghcr.io/catthehacker/ubuntu:runner-22.04
+            echo act -P ubuntu-24.04=ghcr.io/catthehacker/ubuntu:runner-24.04
             exit 1
           fi
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Install depedencies
         run: |
@@ -27,5 +27,8 @@ jobs:
           fi
           sudo apt-get install -y libio-socket-ssl-perl libnet-ssleay-perl netcat
 
-      - name: Run test
+      - name: Test Strings File
+        run: perl -ICPAN t/verifyStrings.pl
+
+      - name: Run Startup Test
         run: bash t/00_smoketest.sh
diff --git a/t/verifyStrings.pl b/t/verifyStrings.pl
new file mode 100644
index 00000000000..c328da72407
--- /dev/null
+++ b/t/verifyStrings.pl
@@ -0,0 +1,50 @@
+#!/usr/bin/env perl
+
+use strict;
+use warnings;
+
+use File::Next;
+use File::Slurp;
+use JSON;
+
+my %translations;
+my (@output, @invalid, $slug);
+
+my $files = File::Next::files( {
+	file_filter => sub { /\bstrings\.txt$/ }
+}, '.' );
+
+while (my $file = $files->()) {
+	foreach (read_file($file)) {
+		my ($lang, $translation);
+		chomp;
+
+		next if /^#/;
+
+		if (/^\t([-A-Z0-9_]{2,5})\t(.*)/) {
+			($lang, $translation) = ($1, $2);
+		} elsif (/^[-A-Z0-9_]+$/) {
+			$slug = $_;
+		} elsif ($_) {
+			push @invalid, "Invalid line ($slug): $_";
+		}
+
+		if ($lang && !exists $translations{$lang}) {
+			$translations{$lang} = {};
+		}
+
+		if ($lang) {
+			$translations{$lang}{$slug} = $translation;
+		}
+	}
+}
+
+foreach my $lang (grep { !$translations{$_}->{LANGUAGE_CHOICES} } keys %translations) {
+	push @invalid, "Invalid language: $lang";
+}
+
+if (scalar(@invalid)) {
+	print "Errors:\n";
+	print join("\n", @invalid);
+	exit 1;
+}