From 9e2f3d59cf249403859916df9756c179753ea7e0 Mon Sep 17 00:00:00 2001
From: Thorsten Töpper <atsutane@freethoughts.de>
Date: Sun, 10 Aug 2025 18:16:07 +0200
Subject: split_for_sort: Split a given file into buckets

The target bucket is decided based on the first X characters of a line.
The bucket name gets a prefix defined as argument and can be sorted
faster on weak hardware. Note: This is just a split alternative.

Real world usage in a shell script with a file in which the first 10
characters are the equal in each line, the following 2 bytes are
evaluated for splitting:

split_for_sort TMPSFS 12 raw_data.txt
for f in TMPSFS ; do
    sort -o "${f}_sorted" -u "${f}"
done
\# Rely on the argument resolution to go with lexical order
cat TMPSFS*_sorted > sorted_data.txt
rm TMPSFS*
---
 include/hex_conversion.h | 96 ++++++++++++++++++++++++++++++++++++++++++++++++
 include/output.h         | 20 ++++++++++
 2 files changed, 116 insertions(+)
 create mode 100644 include/hex_conversion.h
 create mode 100644 include/output.h

(limited to 'include')

diff --git a/include/hex_conversion.h b/include/hex_conversion.h
new file mode 100644
index 0000000..b31216d
--- /dev/null
+++ b/include/hex_conversion.h
@@ -0,0 +1,96 @@
+/*
+ * vim:ts=4:sw=4:expandtab
+ */
+#ifndef HEX_CONVERSION_H
+#define HEX_CONVERSION_H
+
+#include <stdlib.h>
+#include <string.h>
+#include <ctype.h>
+
+#ifdef DEBUGBUILD
+#include "output.h"
+#endif
+
+#define ishex_macro(c) ((c>='0' && c <= '9') || (c>='A' && c <= 'F') || (c>='a' && c <= 'f'))
+
+int convert_line(char *s);
+int ishex(char c);
+char *convert_to_binary(char *hex, char *out);
+char *convert_from_binary(char *bin, size_t l, char *out);
+
+/* short inline functions are fine in header */
+inline int convert_line(char *s) {
+    size_t i = 0, l = 0;
+    if (s == NULL)
+        return -1;
+    l=strlen(s);
+    for (i=0; i<l; i++) {
+        s[i] = toupper(s[i]);
+        if ((s[i] == '\r' && (l-i<3)) || (s[i] == '\n' && i==l-1)) {
+            s[i] = '\0';
+            break;
+        }
+    }
+    return 0;
+}
+
+inline int ishex(char c) {
+    if ((c>='0' && c <= '9') || (c>='A' && c <= 'F') || (c>='a' && c <= 'f')) {
+        return 1;
+    }
+    return 0;
+};
+
+inline char *convert_to_binary(char *hex, char *out) {
+    char tmp[3] = {0,0,0};
+    size_t length, i;
+    if (hex == NULL) return NULL;
+    length=strlen(hex);
+    if ( (length==0) || (length%2 == 1)) return NULL;
+    for (i=0; i<length; i++) {
+        if ( ! ishex_macro(hex[i]) ) {
+#ifdef DEBUGBUILD
+            LOGERR("Incompatible string '%s'\n", hex);
+#endif
+            return NULL;
+        }
+    }
+    if (out == NULL && ((out = calloc((length/2),sizeof(char))) == NULL)) {
+#ifdef DEBUGBUILD
+        LOGERR("ERROR: Failed to allocate %d bytes\n", (length/2));
+#endif
+        return NULL;
+    }
+    for (i=0;i<length;i+=2) {
+        tmp[0] = hex[i];
+        tmp[1] = hex[i+1];
+        out[i/2] = (char) strtol(tmp, NULL, 16);
+    }
+    return out;
+}
+
+/* Use a large buffer and complex method, as with a simple
+ * way there regularly were corrupt results with gcc -O2. */
+inline char *convert_from_binary(char *bin, size_t l, char *out) {
+    char tmp[24];
+    size_t i,pos;
+    if (bin == NULL || l == 0) return NULL;
+    if (out == NULL && (out = calloc(((l*2)+1),sizeof(char))) == NULL) {
+#ifdef DEBUGBUILD
+        LOGERR("ERROR: Failed to allocate %lu bytes\n", ((l*2)+1));
+#endif
+        return NULL;
+    }
+    for (i=0; i<l; i++) {
+        /* Keep in mind this format is not only two characters. */
+        sprintf(tmp, "%02X", (unsigned char)bin[i]);
+        pos=strlen(tmp);
+        out[i*2] = tmp[pos-2];
+        out[(i*2)+1] = tmp[pos-1];
+    }
+    return out;
+}
+
+#endif
+
diff --git a/include/output.h b/include/output.h
new file mode 100644
index 0000000..efc7487
--- /dev/null
+++ b/include/output.h
@@ -0,0 +1,20 @@
+/*
+ * vim:ts=4:sw=4:expandtab
+ */
+#ifndef OUTPUT_H
+#define OUTPUT_H
+
+#include <stdio.h>
+
+#ifndef LOGERR
+#define LOGERR(...) {fprintf(stderr, "[%s:%d] %s: ", __FILE__, __LINE__, __func__); fprintf(stderr, __VA_ARGS__);}
+#endif
+
+#ifdef DEBUGBUILD
+#define DBGTRC(...) LOGERR(__VA_ARGS__)
+#else
+#define DBGTRC(...)
+#endif
+
+#endif
+
-- 
cgit v1.2.3-70-g09d2