Fix download method

Thanks White_Rabbit!

1 files changed, 36 insertions, 10 deletions

@@ -1512,18 +1512,44 @@ extract_links() {
 # `download' works by reading the end of the pipe from `display', which means
 # that sometimes, due to something with the way bash or while or ... something
 # ... chunks the data, sometimes binary data gets corrupted.  This is an area
-# that requires more research.
+# that requires more research.  UPDATE 2022-06-08: White_Rabbit has figured out
+# solution!  From their email:
+#
+# > by the time we're ready to save a non-text/* resource, it's already
+# > corrupted beyond repair. One possibile solution, attached, is to request it
+# > again, presume the reply is 20 and save the data to the filesystem. What do
+# > you think?
+#
+# I think this is great, thanks!!!
 download() {
-        tn="$(mktemp)"
-        log x "Downloading: '$BOLLUX_URL' => '$tn'..."
-        dd status=progress >"$tn"
-        fn="$BOLLUX_DOWNDIR/${BOLLUX_URL##*/}"
-        if [[ -f "$fn" ]]; then
-                log x "Saved '$tn'."
-        elif mv "$tn" "$fn"; then
-                log x "Saved '$fn'."
+        # The binary file has been corrupted by normalize, which strips 0x0d
+        # bytes. Something also drops NULL bytes. So, we'll discard this data
+        cat >/dev/null
+        # Now it's time to re-download the binary file
+        temp_data="$(mktemp)"
+        log x "Downloading: '$BOLLUX_URL' => '$temp_data'..."
+        gemini_request "$BOLLUX_URL" | dd status=progress >"$temp_data"
+
+        # Now $temp_data holds both the header and the data
+        HEADER=$(head -1 "$temp_data")
+        # To get the header length we use ${#HEADER} syntax, but this gives
+        # a bad value because it doesn't count the last byte 0x0A.
+        # We sum 2 because tail wants the first useful byte.
+        let FIRST_BYTE=$((${#HEADER} + 2))
+        temp_name="$(mktemp)"
+        if tail --bytes=+$FIRST_BYTE "$temp_data" >"$temp_name"; then
+                rm "$temp_data"
+        else
+                log error "Error removing the header from '$temp_data'."
+        fi
+
+        final_name="$BOLLUX_DOWNDIR/${BOLLUX_URL##*/}"
+        if [[ -f "$final_name" ]]; then
+                log x "Saved '$temp_name'."
+        elif mv "$temp_name" "$final_name"; then
+                log x "Saved '$final_name'."
         else
-                log error "Error saving '$fn': downloaded to '$tn'."
+                log error "Error saving '$final_name': downloaded to '$temp_name'."
         fi
 }