Add h5ai http backend

[lflare]

What is the purpose of this change?

This change adds support for h5ai, as it does not seem to be properly compatible with the normal HTTP backend.

Verification of functionality,

./rclone copy -vP ":h5ai,url='https://larsjung.de/h5ai/demo/':" /tmp/test/

Note, changes from the original HTTP backend are very minimal, so if there's a better way to do this, I'd be open to modifying my PR.

--- ../http/http.go	2023-12-12 23:27:29.318759323 +0800
+++ h5ai.go	2024-03-03 21:29:57.226883201 +0800
@@ -2,10 +2,12 @@
 //
 // It treats HTML pages served from the endpoint as directory
 // listings, and includes any links found as files.
-package http
+package h5ai
 
 import (
+	"bytes"
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
@@ -27,14 +29,14 @@
 )
 
 var (
-	errorReadOnly = errors.New("http remotes are read only")
+	errorReadOnly = errors.New("h5ai remotes are read only")
 	timeUnset     = time.Unix(0, 0)
 )
 
 func init() {
 	fsi := &fs.RegInfo{
-		Name:        "http",
-		Description: "HTTP",
+		Name:        "h5ai",
+		Description: "H5AI",
 		NewFs:       NewFs,
 		CommandHelp: commandHelp,
 		Options: []fs.Option{{
@@ -436,7 +438,21 @@
 		return nil, fmt.Errorf("internal error: readDir URL %q didn't end in /", URL)
 	}
 	// Do the request
-	req, err := http.NewRequestWithContext(ctx, "GET", URL, nil)
+	payload := &struct {
+		Action string            `json:"action"`
+		Items  map[string]string `json:"items"`
+	}{
+		Action: "get",
+		Items: map[string]string{
+			"href": u.Path,
+			"what": "1",
+		},
+	}
+	buf, err := json.Marshal(payload)
+	if err != nil {
+		return nil, fmt.Errorf("readDir failed: %w", err)
+	}
+	req, err := http.NewRequestWithContext(ctx, "POST", URL, bytes.NewReader(buf))
 	if err != nil {
 		return nil, fmt.Errorf("readDir failed: %w", err)
 	}
@@ -453,15 +469,23 @@
 		return nil, fmt.Errorf("failed to readDir: %w", err)
 	}
 
-	contentType := strings.SplitN(res.Header.Get("Content-Type"), ";", 2)[0]
-	switch contentType {
-	case "text/html":
-		names, err = parse(u, res.Body)
-		if err != nil {
-			return nil, fmt.Errorf("readDir: %w", err)
+	body, err := io.ReadAll(res.Body)
+	type Item struct {
+		Href string `json:"href"`
+	}
+	type Items struct {
+		Items []Item `json:"items"`
+	}
+	items := Items{}
+	err = json.Unmarshal(body, &items)
+	if err != nil {
+		return nil, fmt.Errorf("failed to readDir: %w", err)
+	}
+	for _, v := range items.Items {
+		href, _ := url.QueryUnescape(v.Href)
+		if href != u.Path && strings.HasPrefix(href, u.Path) {
+			names = append(names, strings.TrimPrefix(href, u.Path))
 		}
-	default:
-		return nil, fmt.Errorf("can't parse content type %q", contentType)
 	}
 	return names, nil
 }

Was the change discussed in an issue or in the forum before?

No

Checklist

• I have read the contribution guidelines.
• I have added tests for all changes in this PR if appropriate.
• I have added documentation for the changes if appropriate.
• All commit messages are in house style.
• I'm done, this Pull Request is ready for review :-)

[kapitainsky]

what about documentation? tests?

[ncw]

From looking at your changes I see the main differences from the http backend are

• Requests submitted as JSON via POST
• Directory listings returned as JSON

Perhaps we should have a provider flag like we do in s3 which could have the values other (the default) and h5ai.

What do you think?

[lflare]

what about documentation? tests?

Apologies, I'm not too confident on what's required for either, and I wrote this PR as an afterthought after writing the h5ai backend for my personal use, thinking that it might be of use to someone out there.

I do think that it might be a better idea to implement what @ncw has suggested, given how much of the original http remote I reused for the h5ai remote.

If anyone else wants to clone the branch and take credit for this feature, feel free to do so, let me know and I will close this PR in favor of a better one.