(摘) piping 文件/文本传输工具

声明:内容源自网络,版权归原作者所有。若有侵权请联系我们

Piping是一个轻量级的开源文件传输工具,可自托管,支持使用curl,wget下载。可用于不同设备间传输文本或文件。
Github 地址:https://github.com/nwtgck/piping-server

curl 传递文本
发送端 echo ‘hello,world’ | curl -T - https://ppng.io/hello 接口端 curl https://ppng.io/hello > hello.txt

stream流式获取数据
发送端 seq inf | curl -T - https://ppng.io/seqinf
接口端 curl https://ppng.io/seqinf

传输路径可以随意批定,比如上例中的/hello。
发送方和接收方都可以率先发起传输。

在JavaScript中,可以使用 fetch() 进行传输:

发送
fetch("https://ppng.io/hello",{
    method: "POST",
    body: "hello,world"
});

接收
const res = await fetch("https://ppng.io/hello");
console.log(await res.text());

Piping Server采用的是HTTP流式传输,这意味着可以无限传输任何数据。
下面示例展示如何传输文件夹:

tar c ./mydir | curl -T - https://ppng.io/mypath

curl https://ppng.io/mypath | tar xv

上述例子中,文件夹在上传过程中被打包并且在下载过程中被解压,通过流式传输并不需要其他创建临时文件。

采用如下方式可以轻松进行端对端加密传输:
send: … | openssle aes-256-cbc | curl -T …
get: curl … | openssl aes-256-cbc -d

同时也可以通过压缩减小传输数据大小:
send: … | gzip | curl -T …
get: curl … | zcat

你可以通过管道(pipe)组合完成任何数据的传输。Piping Server的名字就来源于此,因此也可以直译为管道传输服务器。

用来实现一个聊天
curl.exe -T - https://ppng.io/abcd
curl.exe https://ppng.io/abcd
不过看起来命令行对中文支持有问题

指定多个接收方

传输可以拼写多个接收方,通过增加请求参数 ?n=3,可以指定3个接收方。
发送 seq 1000 | curl -T - https://piping.ml/mypath?n=3
接收 curl https://piping.ml/mypath?n=3

公用传输服务器

https://ppng.io
https://piping.glitch.me
https://piping-47q675ro2guv.runkit.sh
https://piping.nwtgck.repl.co
https://ppng.herokuapp.com (注意: Heroku 不支持流式传输)

https://piping-ui.org/ (测试是可以用的)

使用 Docker 自建传输服务器

通过以下命令可以在 http://localhost:8080 运行 Pipe Server: docker run -p 8080:8080 nwtgck/piping-server
后台运行并且自动重启 docker run -p 8080:8080 -d –restart=always nwtgck/piping-server
虽然github.com上面没有直接提供编译后的文件,但README中说了如何通过编译文件自建服务器。
Linux上: curl -L https://github.com/nwtgck/piping-server-pkg/releases/download/v1.12.0/piping-server-pkg-linuxstatic-x64.tar.gz | tar xzvf - ./piping-server-pkg-linuxstatic-x64/piping-server –http-port=8080
其它主机上看这里:https://github.com/nwtgck/piping-server-pkg
https://github.com/nwtgck/piping-server-pkg/releases 这里下载


可以用它来实现程序的自动升级吗?
这里还有国外网友更多的研究: https://dev.to/nwtgck/the-power-of-pure-http-screen-share-real-time-messaging-ssh-and-vnc-5ghc,共享视频,共享屏幕,SSH…

多搜索了一下,果然有Go版本的Piping Server: https://github.com/nwtgck/go-piping-server

自己建立了一个Piping服务器: https://o.scwy.net
因为我使用Caddy,它会自己建立https,自动指向Piping的http服务端。

我用Go版建立了Piping服务器。从使用来看有些差别,比如不支持上面的聊天方式,不支持多个接收方。

复制一下它的关键代码

package piping_server

import (
	"fmt"
	"io"
	"log"
	"mime"
	"mime/multipart"
	"net/http"
	"net/textproto"
	"strconv"
	"sync"
	"sync/atomic"

	"github.com/nwtgck/go-piping-server/version"
)

const (
	reservedPathIndex      = "/"
	reservedPathNoScript   = "/noscript"
	reservedPathVersion    = "/version"
	reservedPathHelp       = "/help"
	reservedPathFaviconIco = "/favicon.ico"
	reservedPathRobotsTxt  = "/robots.txt"
)

var reservedPaths = [...]string{
	reservedPathIndex,
	reservedPathVersion,
	reservedPathHelp,
	reservedPathFaviconIco,
	reservedPathRobotsTxt,
}

const noscriptPathQueryParameterName = "path"

type pipe struct {
	receiverResWriterCh chan http.ResponseWriter
	sendFinishedCh      chan struct{}
	isSenderConnected   uint32 // NOTE: for atomic operation
	isTransferring      uint32 // NOTE: for atomic operation
}

type PipingServer struct {
	pathToPipe map[string]*pipe
	mutex      *sync.Mutex
	logger     *log.Logger
}

func isReservedPath(path string) bool {
	for _, p := range reservedPaths {
		if p == path {
			return true
		}
	}
	return false
}

func NewServer(logger *log.Logger) *PipingServer {
	return &PipingServer{
		pathToPipe: map[string]*pipe{},
		mutex:      new(sync.Mutex),
		logger:     logger,
	}
}

func (s *PipingServer) getPipe(path string) *pipe {
	// Set pipe if not found on the path
	s.mutex.Lock()
	defer s.mutex.Unlock()
	if _, ok := s.pathToPipe[path]; !ok {
		pi := &pipe{
			receiverResWriterCh: make(chan http.ResponseWriter, 1),
			sendFinishedCh:      make(chan struct{}),
			isSenderConnected:   0,
		}
		s.pathToPipe[path] = pi
		return pi
	}
	return s.pathToPipe[path]
}

func transferHeaderIfExists(w http.ResponseWriter, reqHeader textproto.MIMEHeader, header string) {
	values := reqHeader.Values(header)
	if len(values) == 1 {
		w.Header().Add(header, values[0])
	}
}

func getTransferHeaderAndBody(req *http.Request) (textproto.MIMEHeader, io.ReadCloser) {
	mediaType, params, mediaTypeParseErr := mime.ParseMediaType(req.Header.Get("Content-Type"))
	// If multipart upload
	if mediaTypeParseErr == nil && mediaType == "multipart/form-data" {
		multipartReader := multipart.NewReader(req.Body, params["boundary"])
		part, err := multipartReader.NextPart()
		if err != nil {
			// Return normal if multipart error
			return textproto.MIMEHeader(req.Header), req.Body
		}
		return part.Header, part
	}
	return textproto.MIMEHeader(req.Header), req.Body
}

func (s *PipingServer) Handler(resWriter http.ResponseWriter, req *http.Request) {
	s.logger.Printf("%s %s %s", req.Method, req.URL, req.Proto)
	path := req.URL.Path

	if req.Method == "GET" || req.Method == "HEAD" {
		switch path {
		case reservedPathIndex:
			indexPageBytes := []byte(indexPage)
			resWriter.Header().Set("Content-Type", "text/html")
			resWriter.Header().Set("Content-Length", strconv.Itoa(len(indexPageBytes)))
			resWriter.Header().Set("Access-Control-Allow-Origin", "*")
			resWriter.Write(indexPageBytes)
			return
		case reservedPathNoScript:
			noScriptHtmlBytes := []byte(noScriptHtml(req.URL.Query().Get(noscriptPathQueryParameterName)))
			resWriter.Header().Set("Content-Type", "text/html")
			resWriter.Header().Set("Content-Length", strconv.Itoa(len(noScriptHtmlBytes)))
			resWriter.Header().Set("Access-Control-Allow-Origin", "*")
			resWriter.Write(noScriptHtmlBytes)
			return
		case reservedPathVersion:
			versionBytes := []byte(fmt.Sprintf("%s in Go\n", version.Version))
			resWriter.Header().Set("Content-Type", "text/plain")
			resWriter.Header().Set("Content-Length", strconv.Itoa(len(versionBytes)))
			resWriter.Header().Set("Access-Control-Allow-Origin", "*")
			resWriter.Write(versionBytes)
			return
		case reservedPathHelp:
			protocol := "http"
			if req.TLS != nil {
				protocol = "https"
			}
			url := fmt.Sprintf(protocol+"://%s", req.Host)
			helpPageBytes := []byte(helpPage(url))
			resWriter.Header().Set("Content-Type", "text/plain")
			resWriter.Header().Set("Content-Length", strconv.Itoa(len(helpPageBytes)))
			resWriter.Header().Set("Access-Control-Allow-Origin", "*")
			resWriter.Write(helpPageBytes)
			return
		case reservedPathFaviconIco:
			resWriter.Header().Set("Content-Length", "0")
			resWriter.WriteHeader(204)
			return
		case reservedPathRobotsTxt:
			resWriter.Header().Set("Content-Length", "0")
			resWriter.WriteHeader(404)
			return
		}
	}


        // 关键代码如下 -----------------------------------------------------------------
	// TODO: should close if either sender or receiver closes  如果发送方或接收方关闭,则应关闭
	switch req.Method {
	case "GET":
		// If the receiver requests Service Worker registration  如果接收方请求服务人员注册
		// (from: https://speakerdeck.com/masatokinugawa/pwa-study-sw?slide=32)
		if req.Header.Get("Service-Worker") == "script" {
			resWriter.Header().Set("Access-Control-Allow-Origin", "*")
			resWriter.WriteHeader(400)
			resWriter.Write([]byte("[ERROR] Service Worker registration is rejected.\n")) // 服务人员注册被拒绝
			return
		}
		pi := s.getPipe(path)
		// If already get the path or transferring  如果已经获得路径或传输
		if len(pi.receiverResWriterCh) != 0 || atomic.LoadUint32(&pi.isTransferring) == 1 {
			resWriter.Header().Set("Access-Control-Allow-Origin", "*")
			resWriter.WriteHeader(400)
			resWriter.Write([]byte("[ERROR] The number of receivers has reached limits.\n"))
			return
		}
		pi.receiverResWriterCh <- resWriter
		// Wait for finish
		<-pi.sendFinishedCh
	case "POST", "PUT":
		// If reserved path
		if isReservedPath(path) {
			resWriter.Header().Set("Access-Control-Allow-Origin", "*")
			resWriter.WriteHeader(400)
			resWriter.Write([]byte(fmt.Sprintf("[ERROR] Cannot send to the reserved path '%s'. (e.g. '/mypath123')\n", path)))
			return
		}
		// Notify that Content-Range is not supported
		// In the future, resumable upload using Content-Range might be supported
		// ref: https://github.com/httpwg/http-core/pull/653
		if len(req.Header.Values("Content-Range")) != 0 {
			resWriter.Header().Set("Access-Control-Allow-Origin", "*")
			resWriter.WriteHeader(400)
			resWriter.Write([]byte(fmt.Sprintf("[ERROR] Content-Range is not supported for now in %s\n", req.Method)))
			return
		}
		pi := s.getPipe(path)
		// If a sender is already connected
		if !atomic.CompareAndSwapUint32(&pi.isSenderConnected, 0, 1) {
			resWriter.Header().Set("Access-Control-Allow-Origin", "*")
			resWriter.WriteHeader(400)
			resWriter.Write([]byte(fmt.Sprintf("[ERROR] Another sender has been connected on '%s'.\n", path)))
			return
		}
		receiverResWriter := <-pi.receiverResWriterCh
		resWriter.Header().Set("Access-Control-Allow-Origin", "*")

		atomic.StoreUint32(&pi.isTransferring, 1)
		transferHeader, transferBody := getTransferHeaderAndBody(req)
		receiverResWriter.Header()["Content-Type"] = nil // not to sniff
		transferHeaderIfExists(receiverResWriter, transferHeader, "Content-Type")
		transferHeaderIfExists(receiverResWriter, transferHeader, "Content-Length")
		transferHeaderIfExists(receiverResWriter, transferHeader, "Content-Disposition")
		xPipingValues := req.Header.Values("X-Piping")
		if len(xPipingValues) != 0 {
			receiverResWriter.Header()["X-Piping"] = xPipingValues
		}
		receiverResWriter.Header().Set("Access-Control-Allow-Origin", "*")
		if len(xPipingValues) != 0 {
			receiverResWriter.Header().Set("Access-Control-Expose-Headers", "X-Piping")
		}
		receiverResWriter.Header().Set("X-Robots-Tag", "none")
		io.Copy(receiverResWriter, transferBody)
		pi.sendFinishedCh <- struct{}{}
		delete(s.pathToPipe, path)
	case "OPTIONS":
		resWriter.Header().Set("Access-Control-Allow-Origin", "*")
		resWriter.Header().Set("Access-Control-Allow-Methods", "GET, HEAD, POST, PUT, OPTIONS")
		resWriter.Header().Set("Access-Control-Allow-Headers", "Content-Type, Content-Disposition, X-Piping")
		resWriter.Header().Set("Access-Control-Max-Age", "86400")
		resWriter.Header().Set("Content-Length", "0")
		resWriter.WriteHeader(200)
		return
	default:
		resWriter.WriteHeader(405)
		resWriter.Header().Set("Access-Control-Allow-Origin", "*")
		resWriter.Write([]byte(fmt.Sprintf("[ERROR] Unsupported method: %s.\n", req.Method)))
		return
	}
	s.logger.Printf("Transferring %s has finished in %s method.\n", req.URL.Path, req.Method)
}