Hi again Beni,
On Wed, 2022-06-22 at 08:29 +0200, Benedikt Hallinger wrote:
Hi David and thank you for your advice and example.
my pleasure.
I was about to send a slightly improved version of my example code
regarding the use of proxies and the expected content type - see attached
and an extended sample invocation (of course, adapt "myproxy" as needed):
I was about to send a slightly improved version of my example code
regarding the use of proxies and the expected content type - see attached
and an extended sample invocation (of course, adapt "myproxy" as needed):
https_proxy=myproxy ./http_client https://example.com && echo ok
I tried to compile it, run onto errors tough.I just put the file into my openssl source tree, which is on commit:commit 9e86b3815719d29f7bde2294403f97c42ce82a16 (HEAD,origin/openssl-3.0)
I've just tried myself using that commit (and default configuration)
and as expected everything works fine.
and as expected everything works fine.
$ gcc http_client.c -Iinclude -L. -lcrypto -lssl -o http_client/usr/bin/ld: ./libssl.a(libssl-lib-ssl_cert.o): in function`add_uris_recursive':ssl_cert.c:(.text+0x116): undefined reference to `OSSL_STORE_open'/usr/bin/ld: ssl_cert.c:(.text+0x134): undefined reference to`OSSL_STORE_eof'[...]
This issue is pretty surely unrelated to the example code itself
but most likely due to some general build issue you have, such as some inconsistency with pre-installed OpenSSL versions.
Sorry that I do not have the time to provide further aid on such general build issues.
but most likely due to some general build issue you have, such as some inconsistency with pre-installed OpenSSL versions.
Sorry that I do not have the time to provide further aid on such general build issues.
David
Am 2022-06-21 22:52, schrieb David von Oheimb:Hallo Beni,good that you ask.Using the new HTTP client API with TLS (possibly via a proxy) indeedis not easy so far.I'm going to improve this by adding some high-level helper functionsand extending the docs.A good starting point when looking for examples is, as usual, theapplication code in apps/.In this case, there is some pretty useful code in apps/lib/apps.c,but it turns out that the adaptation of app_http_get_asn1() andapp_http_tls_cb()for receiving plain text (rather than ASN.1 encoded data) from theserveris not straightforward because OSSL_HTTP_get() may close the SSL readBIO prematurely.Also the behavior of non-blocking BIOs makes things a little moretricky than expected.Meanwhile I got it working - see the example attached.Example build and usage:gcc http_client.c -Iinclude -L. -lcrypto -lssl -o http_client./http_client https://httpbin.org/ && echo okRegards,DavidOn 20.06.22 10:54, Benedikt Hallinger wrote:Hi there,I currently try to get my hands dirty with C++ and the new HTTPsclientintroduced with OpenSSL 3.However I struggle to get started. My goal is to open a httpssecuredwebsite and download its contents into a std::string for furtherparsing.Does someone on the list know of a small basic example?I imagine that I'm not the first person implementing a HTTPs websiteconnector with OpenSSL 3.Thank you for your support,Beni
#include <openssl/http.h>
#include <openssl/ssl.h>
#include <openssl/err.h>
BIO *bio_err = NULL;
typedef struct app_http_tls_info_st {
const char *server;
const char *port;
int use_proxy;
long timeout;
SSL_CTX *ssl_ctx;
} APP_HTTP_TLS_INFO;
static const char *tls_error_hint(void)
{
unsigned long err = ERR_peek_error();
if (ERR_GET_LIB(err) != ERR_LIB_SSL)
err = ERR_peek_last_error();
if (ERR_GET_LIB(err) != ERR_LIB_SSL)
return NULL; /* likely no TLS error */
switch (ERR_GET_REASON(err)) {
case SSL_R_WRONG_VERSION_NUMBER:
return "The server does not support (a suitable version of) TLS";
case SSL_R_UNKNOWN_PROTOCOL:
return "The server does not support HTTPS";
case SSL_R_CERTIFICATE_VERIFY_FAILED:
return "Cannot authenticate server via its TLS certificate, likely due to mismatch with our trusted TLS certs or missing revocation status";
case SSL_AD_REASON_OFFSET + TLS1_AD_UNKNOWN_CA:
return "Server did not accept our TLS certificate, likely due to mismatch with server's trust anchor or missing revocation status";
case SSL_AD_REASON_OFFSET + SSL3_AD_HANDSHAKE_FAILURE:
return "TLS handshake failure. Possibly the server requires our TLS certificate but did not receive it";
default:
return NULL; /* no hint available for TLS error */
}
}
static BIO *app_http_tls_close(BIO *bio)
{
if (bio != NULL) {
BIO *cbio;
const char *hint = tls_error_hint();
if (hint != NULL)
BIO_printf(bio_err, "%s\n", hint);
(void)ERR_set_mark();
BIO_ssl_shutdown(bio);
cbio = BIO_pop(bio); /* connect+HTTP BIO */
BIO_free(bio); /* SSL BIO */
(void)ERR_pop_to_mark(); /* hide SSL_R_READ_BIO_NOT_SET etc. */
bio = cbio;
}
return bio;
}
/* HTTP callback function that supports TLS connection also via HTTPS proxy */
static BIO *app_http_tls_cb(BIO *bio, void *arg, int connect, int detail)
{
APP_HTTP_TLS_INFO *info = (APP_HTTP_TLS_INFO *)arg;
SSL_CTX *ssl_ctx = info->ssl_ctx;
if (ssl_ctx == NULL) /* not using TLS */
return bio;
if (connect) {
SSL *ssl;
BIO *sbio = NULL;
/* TODO adapt after callback design flaw is fixed, see #17088 */
if ((info->use_proxy
&& !OSSL_HTTP_proxy_connect(bio, info->server, info->port,
NULL, NULL, /* no proxy credentials */
info->timeout, bio_err, "HTTP(S) client"))
|| (sbio = BIO_new(BIO_f_ssl())) == NULL) {
return NULL;
}
if ((ssl = SSL_new(ssl_ctx)) == NULL) {
BIO_free(sbio);
return NULL;
}
/* TODO adapt after callback design flaw is fixed, see #17088 */
SSL_set_tlsext_host_name(ssl, info->server); /* not critical to do */
SSL_set_connect_state(ssl);
BIO_set_ssl(sbio, ssl, BIO_CLOSE);
bio = BIO_push(sbio, bio);
} else if (!detail) { /* disconnect from TLS on error */
bio = app_http_tls_close(bio);
}
return bio;
}
static BIO *app_http_get(const char *url, const char *proxy,
const char *no_proxy, SSL_CTX *ssl_ctx,
const STACK_OF(CONF_VALUE) *headers,
long timeout, const char *expected_content_type)
{
APP_HTTP_TLS_INFO info;
char *server;
char *port;
int use_ssl;
BIO *mem = NULL;
if (url == NULL)
return NULL;
if (!OSSL_HTTP_parse_url(url, &use_ssl, NULL /* userinfo */, &server, &port,
NULL /* port_num, */, NULL, NULL, NULL))
return NULL;
if (use_ssl && ssl_ctx == NULL)
goto end;
info.server = server;
info.port = port;
info.use_proxy = /* workaround for callback design flaw, see #17088 */
OSSL_HTTP_adapt_proxy(proxy, no_proxy, server, use_ssl) != NULL;
info.timeout = timeout;
info.ssl_ctx = ssl_ctx;
mem = OSSL_HTTP_get(url, proxy, no_proxy, NULL /* bio */, NULL /* rbio */,
app_http_tls_cb, &info, 0 /* buf_size */, headers,
expected_content_type, 0 /* expect_asn1 */,
OSSL_HTTP_DEFAULT_MAX_RESP_LEN, timeout);
end:
OPENSSL_free(server);
OPENSSL_free(port);
return mem;
}
int main(int argc, char *argv[])
{
const char *url = argv[1];
const char *proxy = NULL; // use default from environment variables
const char *no_proxy = NULL; // use default from environment variables
SSL_CTX *ssl_ctx = NULL;
const STACK_OF(CONF_VALUE) *headers = NULL;
long timeout = 2; // fail quicky for tests
const char *expected_content_type = NULL; // "text/html; charset=UTF-8";
BIO *bio = NULL;
char buf[1000];
int len;
bio_err = BIO_new_fp(stderr, BIO_NOCLOSE | BIO_FP_TEXT);
if (strncmp(url, "https", 5) == 0)
ssl_ctx = SSL_CTX_new(TLS_client_method());
bio = app_http_get(url, proxy, no_proxy, ssl_ctx, headers, timeout,
expected_content_type);
if (bio != NULL) {
retry:
while ((len = BIO_read(bio, buf, sizeof(buf))) > 0)
printf("%.*s", len, buf);
if (BIO_should_retry(bio))
goto retry;
if (ssl_ctx != NULL)
bio = app_http_tls_close(bio);
}
BIO_free(bio);
ERR_print_errors(bio_err);
BIO_free(bio_err);
return (bio != NULL ? EXIT_SUCCESS : EXIT_FAILURE);
}
