Merge branch 'ma/http-walker-no-partial'
authorJunio C Hamano <gitster@pobox.com>
Tue, 8 May 2018 06:59:35 +0000 (15:59 +0900)
committerJunio C Hamano <gitster@pobox.com>
Tue, 8 May 2018 06:59:35 +0000 (15:59 +0900)
"git http-fetch" (deprecated) had an optional and experimental
"feature" to fetch only commits and/or trees, which nobody used.
This has been removed.

* ma/http-walker-no-partial:
  walker: drop fields of `struct walker` which are always 1
  http-fetch: make `-a` standard behaviour

1  2 
http-fetch.c
remote-curl.c

diff --combined http-fetch.c
@@@ -1,6 -1,6 +1,6 @@@
  #include "cache.h"
  #include "config.h"
 -#include "exec_cmd.h"
 +#include "exec-cmd.h"
  #include "http.h"
  #include "walker.h"
  
@@@ -17,21 -17,13 +17,13 @@@ int cmd_main(int argc, const char **arg
        char *url = NULL;
        int arg = 1;
        int rc = 0;
-       int get_tree = 0;
-       int get_history = 0;
-       int get_all = 0;
        int get_verbosely = 0;
        int get_recover = 0;
  
        while (arg < argc && argv[arg][0] == '-') {
                if (argv[arg][1] == 't') {
-                       get_tree = 1;
                } else if (argv[arg][1] == 'c') {
-                       get_history = 1;
                } else if (argv[arg][1] == 'a') {
-                       get_all = 1;
-                       get_tree = 1;
-                       get_history = 1;
                } else if (argv[arg][1] == 'v') {
                        get_verbosely = 1;
                } else if (argv[arg][1] == 'w') {
                commits = 1;
        }
  
-       if (get_all == 0)
-               warning("http-fetch: use without -a is deprecated.\n"
-                       "In a future release, -a will become the default.");
        if (argv[arg])
                str_end_url_with_slash(argv[arg], &url);
  
@@@ -68,9 -56,6 +56,6 @@@
  
        http_init(NULL, url, 0);
        walker = get_http_walker(url);
-       walker->get_tree = get_tree;
-       walker->get_history = get_history;
-       walker->get_all = get_all;
        walker->get_verbosely = get_verbosely;
        walker->get_recover = get_recover;
  
diff --combined remote-curl.c
@@@ -1,11 -1,10 +1,11 @@@
  #include "cache.h"
  #include "config.h"
  #include "remote.h"
 +#include "connect.h"
  #include "strbuf.h"
  #include "walker.h"
  #include "http.h"
 -#include "exec_cmd.h"
 +#include "exec-cmd.h"
  #include "run-command.h"
  #include "pkt-line.h"
  #include "string-list.h"
@@@ -14,7 -13,6 +14,7 @@@
  #include "credential.h"
  #include "sha1-array.h"
  #include "send-pack.h"
 +#include "protocol.h"
  #include "quote.h"
  
  static struct remote *remote;
@@@ -186,13 -184,12 +186,13 @@@ static int set_option(const char *name
  }
  
  struct discovery {
 -      const char *service;
 +      char *service;
        char *buf_alloc;
        char *buf;
        size_t len;
        struct ref *refs;
        struct oid_array shallow;
 +      enum protocol_version version;
        unsigned proto_git : 1;
  };
  static struct discovery *last_discovery;
  static struct ref *parse_git_refs(struct discovery *heads, int for_push)
  {
        struct ref *list = NULL;
 -      get_remote_heads(-1, heads->buf, heads->len, &list,
 -                       for_push ? REF_NORMAL : 0, NULL, &heads->shallow);
 +      struct packet_reader reader;
 +
 +      packet_reader_init(&reader, -1, heads->buf, heads->len,
 +                         PACKET_READ_CHOMP_NEWLINE |
 +                         PACKET_READ_GENTLE_ON_EOF);
 +
 +      heads->version = discover_version(&reader);
 +      switch (heads->version) {
 +      case protocol_v2:
 +              /*
 +               * Do nothing.  This isn't a list of refs but rather a
 +               * capability advertisement.  Client would have run
 +               * 'stateless-connect' so we'll dump this capability listing
 +               * and let them request the refs themselves.
 +               */
 +              break;
 +      case protocol_v1:
 +      case protocol_v0:
 +              get_remote_heads(&reader, &list, for_push ? REF_NORMAL : 0,
 +                               NULL, &heads->shallow);
 +              break;
 +      case protocol_unknown_version:
 +              BUG("unknown protocol version");
 +      }
 +
        return list;
  }
  
@@@ -285,7 -259,6 +285,7 @@@ static void free_discovery(struct disco
                free(d->shallow.oid);
                free(d->buf_alloc);
                free_refs(d->refs);
 +              free(d->service);
                free(d);
        }
  }
@@@ -317,19 -290,6 +317,19 @@@ static int show_http_message(struct str
        return 0;
  }
  
 +static int get_protocol_http_header(enum protocol_version version,
 +                                  struct strbuf *header)
 +{
 +      if (version > 0) {
 +              strbuf_addf(header, GIT_PROTOCOL_HEADER ": version=%d",
 +                          version);
 +
 +              return 1;
 +      }
 +
 +      return 0;
 +}
 +
  static struct discovery *discover_refs(const char *service, int for_push)
  {
        struct strbuf exp = STRBUF_INIT;
        struct strbuf buffer = STRBUF_INIT;
        struct strbuf refs_url = STRBUF_INIT;
        struct strbuf effective_url = STRBUF_INIT;
 +      struct strbuf protocol_header = STRBUF_INIT;
 +      struct string_list extra_headers = STRING_LIST_INIT_DUP;
        struct discovery *last = last_discovery;
        int http_ret, maybe_smart = 0;
        struct http_get_options http_options;
 +      enum protocol_version version = get_protocol_version_config();
  
        if (last && !strcmp(service, last->service))
                return last;
                strbuf_addf(&refs_url, "service=%s", service);
        }
  
 +      /*
 +       * NEEDSWORK: If we are trying to use protocol v2 and we are planning
 +       * to perform a push, then fallback to v0 since the client doesn't know
 +       * how to push yet using v2.
 +       */
 +      if (version == protocol_v2 && !strcmp("git-receive-pack", service))
 +              version = protocol_v0;
 +
 +      /* Add the extra Git-Protocol header */
 +      if (get_protocol_http_header(version, &protocol_header))
 +              string_list_append(&extra_headers, protocol_header.buf);
 +
        memset(&http_options, 0, sizeof(http_options));
        http_options.content_type = &type;
        http_options.charset = &charset;
        http_options.effective_url = &effective_url;
        http_options.base_url = &url;
 +      http_options.extra_headers = &extra_headers;
        http_options.initial_request = 1;
        http_options.no_cache = 1;
        http_options.keep_error = 1;
                warning(_("redirecting to %s"), url.buf);
  
        last= xcalloc(1, sizeof(*last_discovery));
 -      last->service = service;
 +      last->service = xstrdup(service);
        last->buf_alloc = strbuf_detach(&buffer, &last->len);
        last->buf = last->buf_alloc;
  
                        ;
  
                last->proto_git = 1;
 +      } else if (maybe_smart &&
 +                 last->len > 5 && starts_with(last->buf + 4, "version 2")) {
 +              last->proto_git = 1;
        }
  
        if (last->proto_git)
        strbuf_release(&charset);
        strbuf_release(&effective_url);
        strbuf_release(&buffer);
 +      strbuf_release(&protocol_header);
 +      string_list_clear(&extra_headers, 0);
        last_discovery = last;
        return last;
  }
@@@ -487,7 -426,6 +487,7 @@@ struct rpc_state 
        char *service_url;
        char *hdr_content_type;
        char *hdr_accept;
 +      char *protocol_header;
        char *buf;
        size_t alloc;
        size_t len;
@@@ -674,10 -612,6 +674,10 @@@ static int post_rpc(struct rpc_state *r
        headers = curl_slist_append(headers, needs_100_continue ?
                "Expect: 100-continue" : "Expect:");
  
 +      /* Add the extra Git-Protocol header */
 +      if (rpc->protocol_header)
 +              headers = curl_slist_append(headers, rpc->protocol_header);
 +
  retry:
        slot = get_active_slot();
  
@@@ -818,11 -752,6 +818,11 @@@ static int rpc_service(struct rpc_stat
        strbuf_addf(&buf, "Accept: application/x-%s-result", svc);
        rpc->hdr_accept = strbuf_detach(&buf, NULL);
  
 +      if (get_protocol_http_header(heads->version, &buf))
 +              rpc->protocol_header = strbuf_detach(&buf, NULL);
 +      else
 +              rpc->protocol_header = NULL;
 +
        while (!err) {
                int n = packet_read(rpc->out, NULL, NULL, rpc->buf, rpc->alloc, 0);
                if (!n)
        free(rpc->service_url);
        free(rpc->hdr_content_type);
        free(rpc->hdr_accept);
 +      free(rpc->protocol_header);
        free(rpc->buf);
        strbuf_release(&buf);
        return err;
@@@ -869,9 -797,6 +869,6 @@@ static int fetch_dumb(int nr_heads, str
                targets[i] = xstrdup(oid_to_hex(&to_fetch[i]->old_oid));
  
        walker = get_http_walker(url.buf);
-       walker->get_all = 1;
-       walker->get_tree = 1;
-       walker->get_history = 1;
        walker->get_verbosely = options.verbosity >= 3;
        walker->get_recover = 0;
        ret = walker_fetch(walker, nr_heads, targets, NULL, NULL);
@@@ -1128,202 -1053,6 +1125,202 @@@ static void parse_push(struct strbuf *b
        free(specs);
  }
  
 +/*
 + * Used to represent the state of a connection to an HTTP server when
 + * communicating using git's wire-protocol version 2.
 + */
 +struct proxy_state {
 +      char *service_name;
 +      char *service_url;
 +      struct curl_slist *headers;
 +      struct strbuf request_buffer;
 +      int in;
 +      int out;
 +      struct packet_reader reader;
 +      size_t pos;
 +      int seen_flush;
 +};
 +
 +static void proxy_state_init(struct proxy_state *p, const char *service_name,
 +                           enum protocol_version version)
 +{
 +      struct strbuf buf = STRBUF_INIT;
 +
 +      memset(p, 0, sizeof(*p));
 +      p->service_name = xstrdup(service_name);
 +
 +      p->in = 0;
 +      p->out = 1;
 +      strbuf_init(&p->request_buffer, 0);
 +
 +      strbuf_addf(&buf, "%s%s", url.buf, p->service_name);
 +      p->service_url = strbuf_detach(&buf, NULL);
 +
 +      p->headers = http_copy_default_headers();
 +
 +      strbuf_addf(&buf, "Content-Type: application/x-%s-request", p->service_name);
 +      p->headers = curl_slist_append(p->headers, buf.buf);
 +      strbuf_reset(&buf);
 +
 +      strbuf_addf(&buf, "Accept: application/x-%s-result", p->service_name);
 +      p->headers = curl_slist_append(p->headers, buf.buf);
 +      strbuf_reset(&buf);
 +
 +      p->headers = curl_slist_append(p->headers, "Transfer-Encoding: chunked");
 +
 +      /* Add the Git-Protocol header */
 +      if (get_protocol_http_header(version, &buf))
 +              p->headers = curl_slist_append(p->headers, buf.buf);
 +
 +      packet_reader_init(&p->reader, p->in, NULL, 0,
 +                         PACKET_READ_GENTLE_ON_EOF);
 +
 +      strbuf_release(&buf);
 +}
 +
 +static void proxy_state_clear(struct proxy_state *p)
 +{
 +      free(p->service_name);
 +      free(p->service_url);
 +      curl_slist_free_all(p->headers);
 +      strbuf_release(&p->request_buffer);
 +}
 +
 +/*
 + * CURLOPT_READFUNCTION callback function.
 + * Attempts to copy over a single packet-line at a time into the
 + * curl provided buffer.
 + */
 +static size_t proxy_in(char *buffer, size_t eltsize,
 +                     size_t nmemb, void *userdata)
 +{
 +      size_t max;
 +      struct proxy_state *p = userdata;
 +      size_t avail = p->request_buffer.len - p->pos;
 +
 +
 +      if (eltsize != 1)
 +              BUG("curl read callback called with size = %"PRIuMAX" != 1",
 +                  (uintmax_t)eltsize);
 +      max = nmemb;
 +
 +      if (!avail) {
 +              if (p->seen_flush) {
 +                      p->seen_flush = 0;
 +                      return 0;
 +              }
 +
 +              strbuf_reset(&p->request_buffer);
 +              switch (packet_reader_read(&p->reader)) {
 +              case PACKET_READ_EOF:
 +                      die("unexpected EOF when reading from parent process");
 +              case PACKET_READ_NORMAL:
 +                      packet_buf_write_len(&p->request_buffer, p->reader.line,
 +                                           p->reader.pktlen);
 +                      break;
 +              case PACKET_READ_DELIM:
 +                      packet_buf_delim(&p->request_buffer);
 +                      break;
 +              case PACKET_READ_FLUSH:
 +                      packet_buf_flush(&p->request_buffer);
 +                      p->seen_flush = 1;
 +                      break;
 +              }
 +              p->pos = 0;
 +              avail = p->request_buffer.len;
 +      }
 +
 +      if (max < avail)
 +              avail = max;
 +      memcpy(buffer, p->request_buffer.buf + p->pos, avail);
 +      p->pos += avail;
 +      return avail;
 +}
 +
 +static size_t proxy_out(char *buffer, size_t eltsize,
 +                      size_t nmemb, void *userdata)
 +{
 +      size_t size;
 +      struct proxy_state *p = userdata;
 +
 +      if (eltsize != 1)
 +              BUG("curl read callback called with size = %"PRIuMAX" != 1",
 +                  (uintmax_t)eltsize);
 +      size = nmemb;
 +
 +      write_or_die(p->out, buffer, size);
 +      return size;
 +}
 +
 +/* Issues a request to the HTTP server configured in `p` */
 +static int proxy_request(struct proxy_state *p)
 +{
 +      struct active_request_slot *slot;
 +
 +      slot = get_active_slot();
 +
 +      curl_easy_setopt(slot->curl, CURLOPT_NOBODY, 0);
 +      curl_easy_setopt(slot->curl, CURLOPT_POST, 1);
 +      curl_easy_setopt(slot->curl, CURLOPT_URL, p->service_url);
 +      curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, p->headers);
 +
 +      /* Setup function to read request from client */
 +      curl_easy_setopt(slot->curl, CURLOPT_READFUNCTION, proxy_in);
 +      curl_easy_setopt(slot->curl, CURLOPT_READDATA, p);
 +
 +      /* Setup function to write server response to client */
 +      curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, proxy_out);
 +      curl_easy_setopt(slot->curl, CURLOPT_WRITEDATA, p);
 +
 +      if (run_slot(slot, NULL) != HTTP_OK)
 +              return -1;
 +
 +      return 0;
 +}
 +
 +static int stateless_connect(const char *service_name)
 +{
 +      struct discovery *discover;
 +      struct proxy_state p;
 +
 +      /*
 +       * Run the info/refs request and see if the server supports protocol
 +       * v2.  If and only if the server supports v2 can we successfully
 +       * establish a stateless connection, otherwise we need to tell the
 +       * client to fallback to using other transport helper functions to
 +       * complete their request.
 +       */
 +      discover = discover_refs(service_name, 0);
 +      if (discover->version != protocol_v2) {
 +              printf("fallback\n");
 +              fflush(stdout);
 +              return -1;
 +      } else {
 +              /* Stateless Connection established */
 +              printf("\n");
 +              fflush(stdout);
 +      }
 +
 +      proxy_state_init(&p, service_name, discover->version);
 +
 +      /*
 +       * Dump the capability listing that we got from the server earlier
 +       * during the info/refs request.
 +       */
 +      write_or_die(p.out, discover->buf, discover->len);
 +
 +      /* Peek the next packet line.  Until we see EOF keep sending POSTs */
 +      while (packet_reader_peek(&p.reader) != PACKET_READ_EOF) {
 +              if (proxy_request(&p)) {
 +                      /* We would have an err here */
 +                      break;
 +              }
 +      }
 +
 +      proxy_state_clear(&p);
 +      return 0;
 +}
 +
  int cmd_main(int argc, const char **argv)
  {
        struct strbuf buf = STRBUF_INIT;
                        fflush(stdout);
  
                } else if (!strcmp(buf.buf, "capabilities")) {
 +                      printf("stateless-connect\n");
                        printf("fetch\n");
                        printf("option\n");
                        printf("push\n");
                        printf("check-connectivity\n");
                        printf("\n");
                        fflush(stdout);
 +              } else if (skip_prefix(buf.buf, "stateless-connect ", &arg)) {
 +                      if (!stateless_connect(arg))
 +                              break;
                } else {
                        error("remote-curl: unknown command '%s' from git", buf.buf);
                        return 1;