proto/metadata.proto

// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// The main purpose of mirroring external protos here is to separate API and
// storage protos. The same tag numbers are used for making migration work.

syntax = "proto2";

package mdi.download.internal;

import "google/protobuf/any.proto";
import "google/protobuf/timestamp.proto";
import "transform.proto";

option java_package = "com.google.mobiledatadownload.internal";
option java_outer_classname = "MetadataProto";

// Mirrors mdi.download.ExtraHttpHeader
//
// HTTP headers are described in https://tools.ietf.org/html/rfc7230#section-3.2
// as key:value, where the value may have a whitespace on each end.
message ExtraHttpHeader {
  optional string key = 1;
  optional string value = 2;
}

// This proto is used to store file group metadata on disk for internal use. It
// consists of all fields mirrored from DataFileGroup and an extra field for
// bookkeeping.
//
// The tag number of extra fields should start from 1000 to reserve room for
// growing DataFileGroup.
//
// Next id: 1000
message DataFileGroupInternal {
  // Extra information that is kept on disk.
  //
  // The extension was originally introduced in cl/248813966. We are migrating
  // away from the extension. However, we still need to read from fields in the
  // extension. Reuse the same tag number as the extension number to read from
  // the extension.
  optional DataFileGroupBookkeeping bookkeeping = 248813966;

  // Unique name to identify the group. It should be unique per owner package.
  // In GMSCore, use the module name as the prefix of the group name.
  //
  // Ex: A group name in mdisync module could be named: mdisync-profile-photos.
  //
  // This shouldn't ideally be something like "config", and
  // instead should better define the feature it will be used for.
  //
  // Ex: "icing-language-detection-model", "smart-action-detection-model"
  optional string group_name = 1;

  // The name of the package that owns this group. If this field is left empty,
  // the owner is assumed to be the package name of the host app.
  //
  // The files will only be downloaded onto the device if the owner package is
  // present on the device.
  //
  // Ex: "com.google.android.gms", "com.google.android.apps.bugle"
  optional string owner_package = 6;

  // Client set version number used to identify the file group.
  //
  // Note that this does not uniquely identify the contents of the file group.
  // It simply reflects a snapshot of client config changes.
  // For example: say there's a file group 'language-detector-model' that
  // downloads a different file per user locale.
  // data_file_group {
  //   file_group_name = 'language-detector-model'
  //   file_group_version_number = 1
  //   file {
  //      url = 'en-model'
  //   }
  // }
  // data_file_group {
  //   file_group_name = 'language-detector-model'
  //   file_group_version_number = 1
  //   file {
  //      url = 'es-model'
  //   }
  // }
  // Note that even though the actual contents of the file group are different
  // for each locale, the version is the same because this config was pushed
  // at the same snapshot.
  //
  // Available GMS v18+.
  optional int32 file_group_version_number = 10;

  // DEPRECATED
  // MDD team recommends to use explicit properties instead.
  optional google.protobuf.Any custom_property = 20 [deprecated = true];

  // Custom metadata attached to the file group.
  //
  // This allows clients to include specific metadata about the group for their
  // own processing purposes. The metadata will be stored with the group and
  // accessible when the file group is retrieved.
  //
  // This property should only be used if absolutely necessary. Please consult
  // with <internal>@ if you have questions about this property or a potential
  // use-case.
  optional google.protobuf.Any custom_metadata = 27;

  reserved 21;

  // Mirrors mdi.download.DataFileGroup.AllowedReaders
  enum AllowedReaders {
    ALL_GOOGLE_APPS = 0;
    ONLY_GOOGLE_PLAY_SERVICES = 1;
    ALL_APPS = 2;
  }

  // Defines who is allowed to read this file group. Currently the options are:
  //
  // ALL_GOOGLE_APPS: accessible to all Google 1p Apps.
  // ONLY_GOOGLE_PLAY_SERVICES: accessible to only GMS Core.
  //
  // If this field is not explicitly set it defaults to "ALL_GOOGLE_APPS".
  //
  // Available GMS v20+.
  optional AllowedReaders allowed_readers_enum = 12;

  // Length of time (in seconds) for which a file group version will live after
  // since a newer version became fully downloaded. Clients should set this time
  // to be more than the time in which they call MDD to refresh their data.
  // NOTE: MDD will delete the file group version within a day of this time.
  // Ex: 172800  // 2 Days
  optional int64 stale_lifetime_secs = 3;

  // The timestamp at which this filegroup should be deleted, even if it is
  // still active, specified in seconds since epoch.
  // NOTE: MDD will delete the file group version within a day of this time.
  optional int64 expiration_date_secs = 11;

  // Specify the conditions under which the file group should be downloaded.
  optional DownloadConditions download_conditions = 13;

  // Setting this flag to true will mean that the downloaded files will appear
  // to be in a directory by themselves.
  // The file name/file path of the exposed file will be the filename set in the
  // file.relative_file_path field, OR if that field is empty, the file name
  // from the file.url_to_download field. This enables downloaded files to refer
  // to each other by name.
  // It's invalid to set this flag to true if two files end up with the same
  // file path.
  // Valid on iOS, cMDD, and aMDD.
  //
  // NOTE: For aMDD, this feature is not available if Android Blob Sharing is
  // enabled or if using an API level below 21 (L). If either case is true, this
  // option will be ignored.
  optional bool preserve_filenames_and_isolate_files = 14;

  // List of files in the group.
  repeated DataFile file = 2;

  // Tag for the network traffic to download this file group.
  // Tag space is determined by the host app.
  // For Gmscore, the tag should come from:
  // <internal>
  optional int32 traffic_tag = 16;

  // Extra HTTP headers to apply when downloading all files in the group.
  repeated ExtraHttpHeader group_extra_http_headers = 17;

  reserved 19;

  // Unique identifier of a DataFileGroup config (i.e. a "snapshot") created
  // when using MDD Ingress API.
  optional int64 build_id = 23;

  // A fingerprint allowing clients to identify a DataFileGroup
  // config based on a given set of properties (i.e. a "partition" of
  // any file group properties). This can be used by clients as an exact match
  // for a class of DataFileGroups during targeting or as a compatibility check.
  optional string variant_id = 26;

  // The locales compatible with the file group. This can be different from the
  // device locale.
  //
  // Values in this list may be exact locales (e.g. "en-US") or language-only
  // ("en-*").
  // Example 1: locale = ["en-US"]; // compatible with "en-US" only
  // Example 2: locale = ["en-US", "en-CA"]; // compatible with "en-US" or
  //                                         // "en-CA"
  // Example 3: locale = ["en-*"]; // compatible with all "en" locales
  repeated string locale = 25;

  reserved 28;

  reserved 4, 5, 7, 8, 9, 15, 18, 22, 24;
}

// Mirrors mdi.download.DataFile
//
// A data file represents all the metadata to download the file and then
// manage it on the device.
// Next tag: 22
//
// This should not contain any fields that are marked internal, as we compare
// the protos directly to decide if it is a new version of the file.
// LINT.IfChange(data_file)
message DataFile {
  // A unique identifier of the file within the group, that can be used to
  // get this file from the group.
  // Ex: "language-detection-model"
  optional string file_id = 7;

  // Url from where the file is to be downloaded.
  // Ex: https://www.gstatic.com/group-name/model_1234.zip
  optional string url_to_download = 2;

  // Exact size of the file. This is used to check if there is space available
  // for the file before scheduling the download.
  // The byte_size is optional. If not set, MDD will try with best effort to get
  // the file size using the HTTP HEAD request.
  optional int32 byte_size = 4;

  // Enum for checksum types.
  // NOTE: do not add any new checksum type here, older MDD versions would break
  // otherwise.
  enum ChecksumType {
    // Default checksum is SHA1.
    DEFAULT = 0;

    // No checksum is provided.
    NONE = 1;

    reserved 2 /* SHA256 */;
  }

  optional ChecksumType checksum_type = 15;

  // SHA1 checksum to verify the file before it can be used. This is also used
  // to de-duplicate files between different groups.
  // For most files, this will be the checksum of the file being downloaded.
  // For files with download_transform, this should contain the transform of
  // the file after the transforms have been applied.
  // The checksum is optional. If not set, the checksum_type must be
  // ChecksumType.NONE.
  optional string checksum = 5;

  // The following are <internal> transforms to apply to the downloaded files.
  // Transforms are bi-directional and defined in terms of what they do on
  // write. Since these transforms are applied while reading, their
  // directionality is reversed. Eg, you'll see 'compress' to indicate that the
  // file should be decompressed.

  // These transforms are applied once by MDD after downloading the file.
  // Currently only compress is available.
  // Valid on Android. iOS support is tracked by b/118828045.
  optional mobstore.proto.Transforms download_transforms = 11;

  // If DataFile has download_transforms, this field must be provided with the
  // SHA1 checksum of the file before any transform are applied. The original
  // checksum would also be checked after the download_transforms are applied.
  optional string downloaded_file_checksum = 14;

  // Exact size of the downloaded file. If the DataFile has download transforms
  // like compress and zip, the downloaded file size would be different than
  // the final file size on disk. Client could use
  // this field to track the downloaded file size and calculate the download
  // progress percentage. This field is not used by MDD currently.
  optional int32 downloaded_file_byte_size = 16;

  // These transforms are evaluated by the caller on-the-fly when reading the
  // data with MobStore. Any transforms installed in the caller's MobStore
  // instance is available.
  // Valid on Android. iOS support is tracked by b/118759254.
  optional mobstore.proto.Transforms read_transforms = 12;

  // List of delta files that can be encoded and decoded with base files.
  // If the device has any base file, the delta file which is much
  // smaller will be downloaded instead of the full file.
  // For most clients, only one delta file should be enough. If specifying
  // multiple delta files, they should be in a sequence from the most recent
  // base file to the oldest.
  // This is currently only supported on Android.
  repeated DeltaFile delta_file = 13;

  enum AndroidSharingType {
    // The dataFile isn't available for sharing.
    UNSUPPORTED = 0;

    // If sharing with the Android Blob Sharing Service isn't available, fall
    // back to normal behavior, i.e. download locally.
    ANDROID_BLOB_WHEN_AVAILABLE = 1;
  }

  // Defines whether the file should be shared and how.
  // NOTE: currently this field is only used by aMDD and has no effect on iMDD.
  optional AndroidSharingType android_sharing_type = 17;

  // Enum for android sharing checksum types.
  enum AndroidSharingChecksumType {
    NOT_SET = 0;

    // If the file group should be shared through the Android Blob Sharing
    // Service, the checksum type must be set to SHA256.
    SHA2_256 = 1;
  }

  optional AndroidSharingChecksumType android_sharing_checksum_type = 18;

  // Checksum used to access files through the Android Blob Sharing Service.
  optional string android_sharing_checksum = 19;

  // Relative file path and file name to be preserved within the parent
  // directory when creating symlinks for the file groups that have
  // preserve_filenames_and_isolate_files set to true.
  // This filename should NOT start or end with a '/', and it can not contain
  // the substring '..'.
  // Working example: "subDir/FileName.txt".
  optional string relative_file_path = 20;

  // Custom metadata attached to the file.
  //
  // This allows clients to include specific metadata about the file for their
  // own processing purposes. The metadata will be stored with the file and
  // accessible when the file's file group is retrieved.
  //
  // This property should only be used if absolutely necessary. Please consult
  // with <internal>@ if you have questions about this property or a potential
  // use-case.
  optional google.protobuf.Any custom_metadata = 21;

  reserved 1, 3, 6, 8, 9;
}
// LINT.ThenChange(
//     <internal>,
//     <internal>)

// Mirrors mdi.download.DeltaFile
//
// A delta file represents all the metadata to download for a diff file encoded
// based on a base file
// LINT.IfChange(delta_file)
message DeltaFile {
  // These fields all mirror the similarly-named fields in DataFile.
  optional string url_to_download = 1;
  optional int32 byte_size = 2;
  optional string checksum = 3;

  // Enum of all diff decoders supported
  enum DiffDecoder {
    // Default to have no diff decoder specified, will thrown unsupported
    // exception
    UNSPECIFIED = 0;

    // VcDIFF decoder
    // Generic Differencing and Compression Data Format
    // For more information, please refer to rfc3284
    // The VcDiff decoder for GMS service:
    // <internal>
    VC_DIFF = 1;
  }
  // The diff decoder used to generate full file with delta and base file.
  // For MDD as a GMS service, a VcDiff decoder will be registered and injected
  // in by default. Using MDD as a library, clients need to register and inject
  // in a VcDiff decoder, otherwise, an exception will be thrown.
  optional DiffDecoder diff_decoder = 5;

  // The base file represents to a full file on device. It should contain the
  // bare minimum fields of a DataFile to identify a DataFile on device.
  optional BaseFile base_file = 6;

  reserved 4;
}
// LINT.ThenChange(
//     <internal>,
//     <internal>)

// Mirrors mdi.download.BaseFile
message BaseFile {
  // SHA1 checksum of the base file to identify a file on device. It should
  // match the checksum field of the base file used to generate the delta file.
  optional string checksum = 1;
}

// Mirrors mdi.download.DownloadConditions
//
// Next id: 5
message DownloadConditions {
  // TODO(b/143548753): The first value in an enum must have a specific prefix.
  enum DeviceStoragePolicy {
    // MDD will block download of files in android low storage. Currently MDD
    // doesn't delete the files in case the device reaches low storage
    // after the file has been downloaded.
    BLOCK_DOWNLOAD_IN_LOW_STORAGE = 0;

    // Block download of files only under a lower threshold defined here
    // <internal>
    BLOCK_DOWNLOAD_LOWER_THRESHOLD = 1;

    // Set the storage threshold to an extremely low value when downloading.
    // IMPORTANT: if the download make the device runs out of disk, this could
    // render the device unusable.
    // This should only be used for critical use cases such as privacy
    // violations. Emergency fix should not belong to this category. Please
    // talks to <internal>@ when you want to use this option.
    EXTREMELY_LOW_THRESHOLD = 2;
  }

  // Specify the device storage under which the files should be downloaded.
  // By default, the files will only be downloaded if the device is not in
  // low storage.
  optional DeviceStoragePolicy device_storage_policy = 1;

  // TODO(b/143548753): The first value in an enum must have a specific prefix.
  enum DeviceNetworkPolicy {
    // Only download files on wifi.
    DOWNLOAD_ONLY_ON_WIFI = 0;

    // Allow download on any network including wifi and cellular.
    DOWNLOAD_ON_ANY_NETWORK = 1;

    // Allow downloading only on wifi first, then after a configurable time
    // period set in the field download_first_on_wifi_period_secs below,
    // allow downloading on any network including wifi and cellular.
    DOWNLOAD_FIRST_ON_WIFI_THEN_ON_ANY_NETWORK = 2;
  }

  // Specify the device network under which the files should be downloaded.
  // By default, the files will only be downloaded on wifi.
  //
  // If your feature targets below v20 and want to download on cellular in
  // these versions of gms, also set allow_dowload_without_wifi = true;
  optional DeviceNetworkPolicy device_network_policy = 2;

  // This field will only be used when the
  // DeviceNetworkPolicy = DOWNLOAD_FIRST_ON_WIFI_THEN_ON_ANY_NETWORK
  // MDD will download the file only on wifi for this period of time. If the
  // download was not finished, MDD will download on any network including
  // wifi and cellular.
  // Ex: 604800  // 7 Days
  optional int64 download_first_on_wifi_period_secs = 4;

  // TODO(b/143548753): The first value in an enum must have a specific prefix.
  enum ActivatingCondition {
    // The download is activated as soon the server side config is received and
    // the server configured download conditions are satisfied.
    ALWAYS_ACTIVATED = 0;

    // The download is activated when both server side activation conditions
    // are satisfied and the client has activated the download on device.
    //
    // Clients can activate this group using the activateFileGroup API.
    // <internal>
    DEVICE_ACTIVATED = 1;
  }

  // Specify how the download is activated. By default, the download is
  // activated as soon as server configured activating conditions are satisfied.
  optional ActivatingCondition activating_condition = 3;
}

// This proto contains extra information about a file group for bookkeeping.
// Next tag: 6
message DataFileGroupBookkeeping {
  // The epoch time (seconds since 1/1/1970) at which this stale file group will
  // be deleted.
  optional int64 stale_expiration_date = 1;

  // The timestamp (epoch time, milliseconds since 1/1/1970) that the file group
  // was first received.
  //
  // If this is an update on an existing group, then the timestamp from the old
  // group is used if no files were updated.
  optional int64 group_new_files_received_timestamp = 2;

  // The timestamp (epoch time, milliseconds since 1/1/1970) at which the group
  // was first marked as downloaded.
  optional int64 group_downloaded_timestamp_in_millis = 3;

  // The timestamp (epoch time, milliseconds since 1/1/1970) that MDD starts
  // downloading the file group for the first time.
  optional int64 group_download_started_timestamp_in_millis = 4;

  // The total count of download periodic tasks needed to fully download the
  // file group.
  optional int32 download_started_count = 5;
}

// Key used by mdd to uniquely identify a client group.
message GroupKey {
  // The name of the group.
  optional string group_name = 1;

  // The package name of the group owner. A null value or empty value means
  // that the group is not associated with any package.
  optional string owner_package = 2;

  // The account associated to the file group.
  optional string account = 5;

  // Whether or not all files in a fileGroup have been downloaded.
  optional bool downloaded = 4;

  // The variant id of the group. A null or empty value indicates that the group
  // does not have an associated variant.
  optional string variant_id = 6;

  reserved 3;
}

// Group Key properties that apply to all groups with that key.
message GroupKeyProperties {
  // Whether this group key has been activated on the device.
  optional bool activated_on_device = 1;
}

// SharedFile is a internal data structure of the SharedFileManager.
message SharedFile {
  optional string file_name = 4;
  optional FileStatus file_status = 5;

  // This field will be used to determine if a file can be retrieved from the
  // Android Blob Sharing Service.
  optional bool android_shared = 8;

  // The maximum expiration date found for a downloaded data file. If
  // {@code android_shared} is set to true, this field stores the current lease
  // expiration date. The default value is 0.
  // See <internal> for more details.
  optional int64 max_expiration_date_secs = 9;

  // Checksum used to access files through the Android Blob Sharing Service.
  optional string android_sharing_checksum = 10;

  // If the file is downloaded successfully but fails checksum matching, we will
  // attempt to delete the file so it can be redownloaded from scratch. To
  // prevent unnecessary network bandwidth, we keep track of the number of these
  // attempts in this field and stop after a certain number. (configurable by a
  // download flag).
  optional int32 checksum_mismatch_retry_download_count = 11;

  reserved 1, 2, 3, 6, 7;
}

// Metadata used by
// com.google.android.libraries.mdi.download.MobileDataDownloadManager
message MobileDataDownloadManagerMetadata {
  optional bool mdd_migrated_to_offroad = 1;
  optional int32 reset_trigger = 2;
}

// Metadata used by
// com.google.android.libraries.mdi.download.SharedFileManager
message SharedFileManagerMetadata {
  optional bool migrated_to_new_file_key = 1;
  optional int64 next_file_name = 2;
}

// Collects all data used by
// com.google.android.libraries.mdi.download.internal.Migrations
message MigrationsStore {
  enum FileKeyVersion {
    NEW_FILE_KEY = 0;
    ADD_DOWNLOAD_TRANSFORM = 1;
    USE_CHECKSUM_ONLY = 2;
  }
  optional bool is_migrated_to_new_file_key = 1;
  optional FileKeyVersion current_version = 2;
}

// Collects all data used by
// com.google.android.libraries.mdi.download.internal.FileGroupsMetadata
message FileGroupsMetadataStore {
  // Key must be a serialized GroupKey.
  map<string, DataFileGroupInternal> data_file_groups = 1;
  // Key must be a serialized GroupKey.
  map<string, GroupKeyProperties> group_key_properties = 2;
  repeated DataFileGroupInternal stale_groups = 3;
}

// Collects all data used by
// com.google.android.libraries.mdi.download.internal.SharedFilesMetadata
message SharedFilesMetadataStore {
  // The key must be a serialized NewFileKey.
  map<string, SharedFile> shared_files = 1;
}

enum FileStatus {
  // The file has never been seen before.
  NONE = 0;
  // The file has been subscribed to, but download has not been attempted.
  SUBSCRIBED = 1;
  // The file download is currently in progress.
  DOWNLOAD_IN_PROGRESS = 2;
  // Downloading the file failed.
  DOWNLOAD_FAILED = 3;
  // The file was downloaded completely, and is available for use.
  DOWNLOAD_COMPLETE = 4;
  // The file was corrupted or lost after being successfully downloaded.
  CORRUPTED = 6;
  // Status returned when their is an error while getting the file status.
  // This is never saved on disk.
  INTERNAL_ERROR = 5;
}

// Key used by the SharedFileManager to identify a shared file.
message NewFileKey {
  // These fields all mirror the similarly-named fields in DataFile.
  optional string url_to_download = 1 [deprecated = true];
  optional int32 byte_size = 2 [deprecated = true];
  optional string checksum = 3;
  optional DataFileGroupInternal.AllowedReaders allowed_readers = 4;
  optional mobstore.proto.Transforms download_transforms = 5
      [deprecated = true];
}

// This proto is used to store state for logging. See details at
// <internal>
message LoggingState {
  // The last time maintenance was run. This should be updated every time
  // maintenance is run.
  // Note: the current implementation only uses this to determine the date of
  // the last log event, but in the future we may want more granular
  // measurements for this, so we store the timestamp as-is.
  optional google.protobuf.Timestamp last_maintenance_run_timestamp = 1;

  // File group specific logging state keyed by GroupKey, build id and version
  // number.
  repeated FileGroupLoggingState file_group_logging_state = 2;

  // Set to true once the shared preferences migration is complete.
  // Note: this field isn't strictly necessary at the moment - we could just
  // check that the file_group_logging_state is empty since no one should write
  // to the network usage monitor shared prefs since the migration will be
  // installed at the same cl where the code is removed. However, if we were to
  // add more fields to FileGroupLoggingState, it would be less straightforward
  // to check for migration state - so having this boolean makes it a bit safer.
  optional bool shared_preferences_network_usage_monitor_migration_complete = 3;

  // Info to enable stable sampling. See <internal> for more
  // info. This field will be set by a migration on first access.
  optional SamplingInfo sampling_info = 4;
}

// This proto is used to store state for logging that is specific to a File
// Group. This includes network usage logging and maybe download tiers (for
// <internal>).
message FileGroupLoggingState {
  optional GroupKey group_key = 1;
  optional int64 build_id = 2;
  optional int32 file_group_version_number = 3;
  optional int64 cellular_usage = 4;
  optional int64 wifi_usage = 5;
}

// Next id: 3
message SamplingInfo {
  // Random number generated and persisted on device. This number should not
  // change (unless device storage/mdd is cleared). It is used as a stable
  // identifier to determine whether MDD should log events.
  optional int64 stable_log_sampling_salt = 1;

  // When the stable_log_sampling_salt was first set. This will be used during
  // roll out to determine which devices have enabled stable sampling for a
  // sufficient time period.
  optional google.protobuf.Timestamp log_sampling_salt_set_timestamp = 2;
}