socket

 1import socket
 2import select
 3import gc
 4import errno
 5
 6URLS = [
 7    "http://httpbin.org/uuid",
 8    "http://httpbin.org/user-agent",
 9    "http://httpbin.org/headers",
10]
11
12TIMEOUT_MS = 5000
13MAX_RECV = 1024
14
15
16def parse_url(url):
17    url = url[7:]
18    if "/" not in url:
19        host, path = url, "/"
20    else:
21        host, path = url.split("/", 1)
22        path = "/" + path
23
24    if ":" in host:
25        host, port = host.split(":")
26        port = int(port)
27    else:
28        port = 80
29
30    return host, port, path
31
32
33def make_request(host, path):
34    return (
35        f"GET {path} HTTP/1.1\r\n" f"Host: {host}\r\n" "Connection: close\r\n" "\r\n"
36    ).encode()
37
38
39poller = select.poll()
40conns = {}
41
42for url in URLS:
43    host, port, path = parse_url(url)
44    sock = socket.socket()
45    sock.setblocking(False)
46
47    try:
48        sock.connect((host, port))  # return EINPROGRESS
49    except OSError as e:
50        if e.errno != errno.EINPROGRESS:
51            print("connect fail", host, e)
52            continue
53
54    poller.register(sock, select.POLLOUT)
55    conns[sock] = {
56        "host": host,
57        "path": path,
58        "state": "connecting",
59        "buf": b"",
60    }
61
62results = []
63
64while conns:
65    for sock, ev in poller.ipoll(TIMEOUT_MS):
66        c = conns[sock]
67
68        if ev & select.POLLOUT:  # connect ok
69            poller.modify(sock, select.POLLIN)
70            c["state"] = "send"
71            sock.send(make_request(c["host"], c["path"]))
72        elif ev & select.POLLIN:  # has data to read
73            data = sock.recv(MAX_RECV)
74            if not data:
75                poller.unregister(sock)
76                sock.close()
77                print("done", c["host"], len(c["buf"]))
78                results.append((c["host"], c["buf"]))
79                del conns[sock]
80                continue
81
82            c["buf"] += data
83        elif ev & (select.POLLERR | select.POLLHUP):
84            print("error", c["host"])
85            poller.unregister(sock)
86            sock.close()
87            del conns[sock]
88
89gc.collect()
90
91for host, buf in results:
92    print(host, "->", buf[:200], b"...")