An API is returning to me this random google bucket url when trying to upload larger files to it:

/...-storage-bucket-prod/google-oauth2%.../...?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=blob-creator-reader%40tctn-eu-services-979d.iam.gserviceaccount%2...%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20241116T214909Z&X-Goog-Expires=59&X-Goog-Signature=...X-Goog-SignedHeaders=host%3Bx-goog-content-length-range%3Bx-goog-hash

I looked at trying to PUT to it, I even tried to give it host, x-goog-content-length-range, x-goog-hash for each chunk as I go through them, but it keeps returning the same error no matter what I do:

host:storage.googleapis
x-goog-content-length-range:0,16383
x-goog-hash:crc32c=ZVBtjQ==

host;x-goog-content-length-range;x-goog-hash
UNSIGNED-PAYLOAD</CanonicalRequest></Error>

My code:

def calculate_crc32c(data: bytes):
    return base64.b64encode(crc32c(data).to_bytes(4, 'big')).decode('utf-8')


def put_file(api: 'API', file: 'File', data: bytes):
    checksum_bs4 = calculate_crc32c(data)
    content_length = len(data)

    position = 0

    def file_chunk_generator(chunk_size=16384):
        nonlocal sync_event, data, content_length, position
        for i in range(0, len(data), chunk_size):
            chunk = data[i:i + chunk_size]
            yield chunk

    with httpx.Client(http2=False) as request:
        response = request.put(
            FILES_URL.format(api.document_storage_uri, file.hash),
            content=file_chunk_generator(),
            headers=(headers := {
                **api.session.headers,
                'content-length': str(content_length),
                'content-type': 'application/octet-stream',
                'x-goog-hash': f'crc32c={checksum_bs4}',
            })
        )

    if response.status_code == 302:
        position = 0
        print("Full google upload detected, continuing")
        url = response.headers.get("Location")
        print(url)
        with httpx.Client(http2=False) as request:

            for chunk in file_chunk_generator():
                print(f'{position},{position + len(chunk) - 1}')
                response = request.put(
                    url,
                    content=chunk,
                    headers={
                        'host': 'storage.googleapis',
                        'x-goog-content-length-range': f'{position},{(position + len(chunk) -1)}',
                        'x-goog-hash': f'crc32c={calculate_crc32c(chunk)}',
                        'Content-Length': str(len(chunk)),
                    }
                )
                if response.status_code != 200:
                    break
                else:
                    print("SUCCESS!!!")

    if response.status_code != 200:
        raise Exception(f"Put file failed - {response.status_code}\n{response.text}")
    else:
        print(file.uuid, "uploaded")

An API is returning to me this random google bucket url when trying to upload larger files to it:

https://storage.googleapis/...-storage-bucket-prod/google-oauth2%.../...?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=blob-creator-reader%40tctn-eu-services-979d.iam.gserviceaccount%2...%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20241116T214909Z&X-Goog-Expires=59&X-Goog-Signature=...X-Goog-SignedHeaders=host%3Bx-goog-content-length-range%3Bx-goog-hash

I looked at trying to PUT to it, I even tried to give it host, x-goog-content-length-range, x-goog-hash for each chunk as I go through them, but it keeps returning the same error no matter what I do:

host:storage.googleapis
x-goog-content-length-range:0,16383
x-goog-hash:crc32c=ZVBtjQ==

host;x-goog-content-length-range;x-goog-hash
UNSIGNED-PAYLOAD</CanonicalRequest></Error>

My code:

def calculate_crc32c(data: bytes):
    return base64.b64encode(crc32c(data).to_bytes(4, 'big')).decode('utf-8')


def put_file(api: 'API', file: 'File', data: bytes):
    checksum_bs4 = calculate_crc32c(data)
    content_length = len(data)

    position = 0

    def file_chunk_generator(chunk_size=16384):
        nonlocal sync_event, data, content_length, position
        for i in range(0, len(data), chunk_size):
            chunk = data[i:i + chunk_size]
            yield chunk

    with httpx.Client(http2=False) as request:
        response = request.put(
            FILES_URL.format(api.document_storage_uri, file.hash),
            content=file_chunk_generator(),
            headers=(headers := {
                **api.session.headers,
                'content-length': str(content_length),
                'content-type': 'application/octet-stream',
                'x-goog-hash': f'crc32c={checksum_bs4}',
            })
        )

    if response.status_code == 302:
        position = 0
        print("Full google upload detected, continuing")
        url = response.headers.get("Location")
        print(url)
        with httpx.Client(http2=False) as request:

            for chunk in file_chunk_generator():
                print(f'{position},{position + len(chunk) - 1}')
                response = request.put(
                    url,
                    content=chunk,
                    headers={
                        'host': 'storage.googleapis',
                        'x-goog-content-length-range': f'{position},{(position + len(chunk) -1)}',
                        'x-goog-hash': f'crc32c={calculate_crc32c(chunk)}',
                        'Content-Length': str(len(chunk)),
                    }
                )
                if response.status_code != 200:
                    break
                else:
                    print("SUCCESS!!!")

    if response.status_code != 200:
        raise Exception(f"Put file failed - {response.status_code}\n{response.text}")
    else:
        print(file.uuid, "uploaded")

• python
• google-cloud-storage

• It's a Google Signed URL. See using Signed URLs with resumable uploads – DazWilkin Commented Nov 16, 2024 at 22:13
• @DazWilkin then why is it not happy with my headers? – RedTTG Commented Nov 16, 2024 at 23:23
• You will want to POST to the Signed URL with Content-Type and Content-Length headers. If the service will accept resumable uploads, you'll be given a different URL to PUT chunks too. – DazWilkin Commented Nov 17, 2024 at 16:32
• Your question is not a minimal repro and would benefit from including specific details of "the same error" and possibly the API that you're using that's generating the Signed URL. – DazWilkin Commented Nov 17, 2024 at 16:33

1 Answer 1

The API was passing x-goog-content-length of 0,7GB that was the required header, not the chunk length