Merge "Build license_checker and test_mapping with Soong." into main
diff --git a/tools/crate-updater/src/main.rs b/tools/crate-updater/src/main.rs
index 107788a..0b2afc1 100644
--- a/tools/crate-updater/src/main.rs
+++ b/tools/crate-updater/src/main.rs
@@ -13,11 +13,9 @@
// limitations under the License.
use std::{
- collections::BTreeSet,
path::{Path, PathBuf},
process::{Command, ExitStatus, Output},
str::from_utf8,
- sync::LazyLock,
};
use anyhow::{bail, Result};
@@ -268,36 +266,6 @@
Ok(())
}
-#[rustfmt::skip]
-static DENYLIST: LazyLock<BTreeSet<&'static str>> = LazyLock::new(|| {
- BTreeSet::from([
- // Paired crates that need to be updated together
- "async-stream",
- "async-stream-impl",
- "clap",
- "clap_builder",
- "linkme",
- "linkme-impl",
- "pin-project",
- "pin-project-internal",
- "protobuf-support",
- "protobuf",
- "ptr_meta",
- "ptr_meta_derive",
- "regex",
- "regex-automata",
- "regex-syntax",
- "serde_derive",
- "serde",
- "thiserror-impl",
- "thiserror",
- "tikv-jemalloc-sys",
- "tikv-jemallocator",
- "zerocopy-derive",
- "zerocopy",
- ])
-});
-
fn main() -> Result<()> {
let args = Cli::parse();
if !args.android_root.is_absolute() {
@@ -325,10 +293,6 @@
for suggestion in get_suggestions(&monorepo_path)? {
let crate_name = suggestion.name.as_str();
let version = suggestion.version.as_str();
- if DENYLIST.contains(crate_name) {
- println!("Skipping {crate_name} (on deny list)");
- continue;
- }
if updates_tried.contains(crate_name, version) {
println!("Skipping {crate_name} (already attempted recently)");
continue;
diff --git a/tools/external_crates/Cargo.toml b/tools/external_crates/Cargo.toml
index 4ee0bfd..4e4c644 100644
--- a/tools/external_crates/Cargo.toml
+++ b/tools/external_crates/Cargo.toml
@@ -6,7 +6,6 @@
"google_metadata",
"license_checker",
"name_and_version",
- "name_and_version_proc_macros",
"repo_config",
"rooted_path",
"test_mapping",
diff --git a/tools/external_crates/cargo_embargo.json b/tools/external_crates/cargo_embargo.json
index 32e2350..6000c7d 100644
--- a/tools/external_crates/cargo_embargo.json
+++ b/tools/external_crates/cargo_embargo.json
@@ -20,9 +20,6 @@
"name_and_version": {
"device_supported": false
},
- "name_and_version_proc_macros": {
- "device_supported": false
- },
"repo_config": {
"device_supported": false
},
diff --git a/tools/external_crates/crate_config/src/lib.rs b/tools/external_crates/crate_config/src/lib.rs
index b6c978b..4e8e0b7 100644
--- a/tools/external_crates/crate_config/src/lib.rs
+++ b/tools/external_crates/crate_config/src/lib.rs
@@ -23,6 +23,9 @@
pub struct CrateConfig {
#[serde(default, skip_serializing_if = "Vec::is_empty")]
deletions: Vec<String>,
+
+ #[serde(default, skip_serializing_if = "Vec::is_empty")]
+ update_with: Vec<String>,
}
#[allow(missing_docs)]
@@ -52,6 +55,10 @@
pub fn deletions(&self) -> impl Iterator<Item = &str> {
self.deletions.iter().map(|d| d.as_str())
}
+ /// Get an iterator of crates that also need to be updated at the same time as this crate.
+ pub fn update_with(&self) -> impl Iterator<Item = &str> {
+ self.update_with.iter().map(|d| d.as_str())
+ }
}
#[cfg(test)]
diff --git a/tools/external_crates/crate_tool/Cargo.toml b/tools/external_crates/crate_tool/Cargo.toml
index d49dfd0..5f27d90 100644
--- a/tools/external_crates/crate_tool/Cargo.toml
+++ b/tools/external_crates/crate_tool/Cargo.toml
@@ -29,7 +29,6 @@
google_metadata = { path = "../google_metadata"}
license_checker = { path = "../license_checker", features = ["fuzzy_content_match"] }
name_and_version = { path = "../name_and_version" }
-name_and_version_proc_macros = { path = "../name_and_version_proc_macros" }
repo_config = { path = "../repo_config" }
rooted_path = { path = "../rooted_path" }
test_mapping = { path = "../test_mapping" }
diff --git a/tools/external_crates/crate_tool/src/crate_collection.rs b/tools/external_crates/crate_tool/src/crate_collection.rs
index f1255f3..50fb29a 100644
--- a/tools/external_crates/crate_tool/src/crate_collection.rs
+++ b/tools/external_crates/crate_tool/src/crate_collection.rs
@@ -13,23 +13,18 @@
// limitations under the License.
use name_and_version::{NameAndVersion, NameAndVersionMap, NamedAndVersioned};
-use name_and_version_proc_macros::NameAndVersionMap;
use rooted_path::RootedPath;
-use std::{
- collections::HashSet,
- path::{Path, PathBuf},
-};
+use std::path::{Path, PathBuf};
use anyhow::{anyhow, Result};
-use semver::Version;
use walkdir::WalkDir;
use crate::{crate_type::Crate, CrateError};
use std::collections::BTreeMap;
-#[derive(NameAndVersionMap, Debug)]
+#[derive(Debug)]
pub struct CrateCollection {
crates: BTreeMap<NameAndVersion, Crate>,
repo_root: PathBuf,
@@ -64,4 +59,22 @@
}
Ok(())
}
+ pub fn len(&self) -> usize {
+ self.crates.len()
+ }
+ pub fn get(&self, nv: &dyn NamedAndVersioned) -> Option<&Crate> {
+ self.crates.get(nv)
+ }
+ pub fn values(&self) -> impl Iterator<Item = &Crate> {
+ self.crates.values()
+ }
+ pub fn get_versions(
+ &self,
+ crate_name: impl AsRef<str>,
+ ) -> Box<dyn Iterator<Item = (&NameAndVersion, &Crate)> + '_> {
+ self.crates.get_versions(crate_name.as_ref())
+ }
+ pub fn contains_crate(&self, crate_name: impl AsRef<str>) -> bool {
+ self.crates.contains_name(crate_name.as_ref())
+ }
}
diff --git a/tools/external_crates/crate_tool/src/managed_crate.rs b/tools/external_crates/crate_tool/src/managed_crate.rs
index c63d089..68bb4a0 100644
--- a/tools/external_crates/crate_tool/src/managed_crate.rs
+++ b/tools/external_crates/crate_tool/src/managed_crate.rs
@@ -237,11 +237,11 @@
}
pub fn fix_test_mapping(&self) -> Result<()> {
let mut tm = TestMapping::read(self.android_crate_path().clone())?;
- println!("{}", self.name());
let mut changed = tm.fix_import_paths();
changed |= tm.add_new_tests_to_postsubmit()?;
changed |= tm.remove_unknown_tests()?;
if changed {
+ println!("Updating TEST_MAPPING for {}", self.name());
tm.write()?;
}
Ok(())
@@ -443,6 +443,7 @@
writeback |= true;
}
if writeback {
+ println!("Updating METADATA for {}", staged.name());
metadata.write()?;
}
}
diff --git a/tools/external_crates/crate_tool/src/managed_repo.rs b/tools/external_crates/crate_tool/src/managed_repo.rs
index d0c9e0b..0057ae4 100644
--- a/tools/external_crates/crate_tool/src/managed_repo.rs
+++ b/tools/external_crates/crate_tool/src/managed_repo.rs
@@ -29,10 +29,10 @@
use google_metadata::GoogleMetadata;
use itertools::Itertools;
use license_checker::find_licenses;
-use name_and_version::{NameAndVersionMap, NameAndVersionRef, NamedAndVersioned};
+use name_and_version::{NameAndVersion, NameAndVersionMap, NameAndVersionRef, NamedAndVersioned};
use repo_config::RepoConfig;
use rooted_path::RootedPath;
-use semver::Version;
+use semver::{Version, VersionReq};
use serde::Serialize;
use spdx::Licensee;
@@ -106,7 +106,6 @@
let cc = self.legacy_crates_for(crate_name)?;
let nv = NameAndVersionRef::new(crate_name, v);
Ok(cc
- .map_field()
.get(&nv as &dyn NamedAndVersioned)
.ok_or(anyhow!("Failed to find crate {} v{}", crate_name, v))?
.path()
@@ -165,7 +164,7 @@
let krate = match version {
Some(v) => {
let nv = NameAndVersionRef::new(crate_name, v);
- match cc.map_field().get(&nv as &dyn NamedAndVersioned) {
+ match cc.get(&nv as &dyn NamedAndVersioned) {
Some(k) => k,
None => {
return Err(anyhow!("Did not find crate {} v{}", crate_name, v));
@@ -173,7 +172,7 @@
}
}
None => {
- if cc.map_field().len() != 1 {
+ if cc.len() != 1 {
return Err(anyhow!(
"Expected a single crate version for {}, but found {}. Specify a version with --versions={}=<version>",
crate_name,
@@ -181,7 +180,7 @@
crate_name
));
}
- cc.map_field().values().next().unwrap()
+ cc.values().next().unwrap()
}
};
println!("Found {} v{} in {}", krate.name(), krate.version(), krate.path());
@@ -428,12 +427,12 @@
let mut found_problems = false;
for (dep, req) in version.android_deps_with_version_reqs() {
println!("Found dep {}", dep.crate_name());
- let cc = if managed_crates.contains_name(dep.crate_name()) {
+ let cc = if managed_crates.contains_crate(dep.crate_name()) {
&managed_crates
} else {
&legacy_crates
};
- if !cc.contains_name(dep.crate_name()) {
+ if !cc.contains_crate(dep.crate_name()) {
found_problems = true;
println!(
" Dep {} {} has not been imported to Android",
@@ -701,7 +700,7 @@
let mut cc = self.new_cc();
cc.add_from(self.managed_dir().rel())?;
- for krate in cc.map_field().values() {
+ for krate in cc.values() {
let cio_crate = self.crates_io.get_crate(krate.name())?;
let upgrades =
cio_crate.versions_gt(krate.version()).map(|v| v.version()).collect::<Vec<_>>();
@@ -776,12 +775,12 @@
// * If a dep is missing, but the same dep exists for the current version of the crate, it's probably not actually necessary.
// * Use relaxed version requirements, treating 0.x and 0.y as compatible, even though they aren't according to semver rules.
for (dep, req) in version.android_deps_with_version_reqs() {
- let cc = if managed_crates.contains_name(dep.crate_name()) {
+ let cc = if managed_crates.contains_crate(dep.crate_name()) {
&managed_crates
} else {
&legacy_crates
};
- if !cc.contains_name(dep.crate_name()) {
+ if !cc.contains_crate(dep.crate_name()) {
found_problems = true;
println!(
" Dep {} {} has not been imported to Android",
@@ -831,7 +830,7 @@
managed_crates.add_from(self.managed_dir().rel())?;
let legacy_crates = self.legacy_crates()?;
- for krate in managed_crates.map_field().values() {
+ for krate in managed_crates.values() {
let cio_crate = self.crates_io.get_crate(krate.name())?;
let base_version = cio_crate.get_version(krate.version());
@@ -869,7 +868,7 @@
if !dep.is_changed_dep(&base_deps) {
return false;
}
- let cc = if managed_crates.contains_name(dep.crate_name()) {
+ let cc = if managed_crates.contains_crate(dep.crate_name()) {
&managed_crates
} else {
&legacy_crates
@@ -908,12 +907,55 @@
Ok(())
}
pub fn update(&self, crate_name: impl AsRef<str>, version: impl AsRef<str>) -> Result<()> {
- let pseudo_crate = self.pseudo_crate();
+ let crate_name = crate_name.as_ref();
let version = Version::parse(version.as_ref())?;
- let nv = NameAndVersionRef::new(crate_name.as_ref(), &version);
- pseudo_crate.remove(&crate_name)?;
- pseudo_crate.cargo_add(&nv)?;
- self.regenerate([&crate_name].iter(), true)?;
+
+ let pseudo_crate = self.pseudo_crate();
+ let managed_crate = self.managed_crate_for(crate_name)?;
+ let mut crate_updates = vec![NameAndVersion::new(crate_name.to_string(), version.clone())];
+
+ let cio_crate = self.crates_io.get_crate(crate_name)?;
+ let cio_crate_version = cio_crate
+ .get_version(&version)
+ .ok_or(anyhow!("Could not find {crate_name} {version} on crates.io"))?;
+
+ for dependent_crate_name in managed_crate.config().update_with() {
+ let dep = cio_crate_version
+ .dependencies()
+ .iter()
+ .find(|dep| dep.crate_name() == dependent_crate_name)
+ .ok_or(anyhow!(
+ "Could not find crate {dependent_crate_name} as a dependency of {crate_name}"
+ ))?;
+ let req = VersionReq::parse(dep.requirement())?;
+ let dep_cio_crate = self.crates_io.get_crate(dependent_crate_name)?;
+ let version = dep_cio_crate
+ .safe_versions()
+ .find(|v| {
+ if let Ok(parsed_version) = Version::parse(v.version()) {
+ req.matches(&parsed_version)
+ } else {
+ false
+ }
+ })
+ .ok_or(anyhow!(
+ "Failed to find a version of {dependent_crate_name} that satisfies {}",
+ dep.requirement()
+ ))?;
+ println!("Also updating {dependent_crate_name} to {}", version.version());
+ crate_updates.push(NameAndVersion::new(
+ dependent_crate_name.to_string(),
+ Version::parse(version.version())?,
+ ));
+ }
+
+ for nv in &crate_updates {
+ pseudo_crate.remove(nv.name())?;
+ }
+ for nv in &crate_updates {
+ pseudo_crate.cargo_add(nv)?;
+ }
+ self.regenerate(crate_updates.iter().map(|nv| nv.name()), true)?;
Ok(())
}
pub fn init(&self) -> Result<()> {
diff --git a/tools/external_crates/license_checker/src/content_checker.rs b/tools/external_crates/license_checker/src/content_checker.rs
index 7e3a144..6f37105 100644
--- a/tools/external_crates/license_checker/src/content_checker.rs
+++ b/tools/external_crates/license_checker/src/content_checker.rs
@@ -82,6 +82,7 @@
("Unlicense", include_str!("licenses/Unlicense.txt")),
("Zlib", include_str!("licenses/Zlib.txt")),
("OpenSSL", include_str!("licenses/OpenSSL.txt")),
+ ("NCSA", include_str!("licenses/NCSA.txt")),
]
.into_iter()
.map(|(req, tokens)| {
diff --git a/tools/external_crates/license_checker/src/licenses/NCSA.txt b/tools/external_crates/license_checker/src/licenses/NCSA.txt
new file mode 100644
index 0000000..2df0b9b
--- /dev/null
+++ b/tools/external_crates/license_checker/src/licenses/NCSA.txt
@@ -0,0 +1,26 @@
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal with
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+ * Redistributions of source code must retain the above copyright notice,
+ this list of conditions and the following disclaimers.
+
+ * Redistributions in binary form must reproduce the above copyright notice,
+ this list of conditions and the following disclaimers in the
+ documentation and/or other materials provided with the distribution.
+
+ * Neither the names of the LLVM Team, University of Illinois at
+ Urbana-Champaign, nor the names of its contributors may be used to
+ endorse or promote products derived from this Software without specific
+ prior written permission.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE
+SOFTWARE.
\ No newline at end of file
diff --git a/tools/external_crates/name_and_version_proc_macros/Android.bp b/tools/external_crates/name_and_version_proc_macros/Android.bp
deleted file mode 100644
index aadadae..0000000
--- a/tools/external_crates/name_and_version_proc_macros/Android.bp
+++ /dev/null
@@ -1,23 +0,0 @@
-// This file is generated by cargo_embargo.
-// Do not modify this file after the first "rust_*" or "genrule" module
-// because the changes will be overridden on upgrade.
-// Content before the first "rust_*" or "genrule" module is preserved.
-
-package {
- default_team: "trendy_team_android_rust",
- default_applicable_licenses: ["Android-Apache-2.0"],
-}
-
-rust_proc_macro {
- name: "libname_and_version_proc_macros",
- crate_name: "name_and_version_proc_macros",
- cargo_env_compat: true,
- cargo_pkg_version: "0.1.0",
- crate_root: "src/lib.rs",
- edition: "2021",
- rustlibs: [
- "libproc_macro2",
- "libquote",
- "libsyn",
- ],
-}
diff --git a/tools/external_crates/name_and_version_proc_macros/Cargo.toml b/tools/external_crates/name_and_version_proc_macros/Cargo.toml
deleted file mode 100644
index 1f72836..0000000
--- a/tools/external_crates/name_and_version_proc_macros/Cargo.toml
+++ /dev/null
@@ -1,12 +0,0 @@
-[package]
-name = "name_and_version_proc_macros"
-version = "0.1.0"
-edition = "2021"
-
-[lib]
-proc-macro = true
-
-[dependencies]
-proc-macro2 = "1"
-syn = "2.0"
-quote = "1.0"
diff --git a/tools/external_crates/name_and_version_proc_macros/src/lib.rs b/tools/external_crates/name_and_version_proc_macros/src/lib.rs
deleted file mode 100644
index 832b2c1..0000000
--- a/tools/external_crates/name_and_version_proc_macros/src/lib.rs
+++ /dev/null
@@ -1,124 +0,0 @@
-// Copyright (C) 2023 The Android Open Source Project
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//! Derive the NameAndVersionMap trait for a struct with a suitable map field.
-
-use syn::{parse_macro_input, DeriveInput, Error};
-
-/// Derive the NameAndVersionMap trait for a struct with a suitable map field.
-#[proc_macro_derive(NameAndVersionMap)]
-pub fn derive_name_and_version_map(input: proc_macro::TokenStream) -> proc_macro::TokenStream {
- let input = parse_macro_input!(input as DeriveInput);
- name_and_version_map::expand(input).unwrap_or_else(Error::into_compile_error).into()
-}
-
-mod name_and_version_map {
- use proc_macro2::TokenStream;
- use quote::quote;
- use syn::{
- Data, DataStruct, DeriveInput, Error, Field, GenericArgument, PathArguments, Result, Type,
- };
-
- pub(crate) fn expand(input: DeriveInput) -> Result<TokenStream> {
- let name = &input.ident;
- let (impl_generics, ty_generics, where_clause) = input.generics.split_for_impl();
-
- let mapfield = get_map_field(get_struct(&input)?)?;
- let mapfield_name = mapfield
- .ident
- .as_ref()
- .ok_or(Error::new_spanned(mapfield, "mapfield ident is none"))?;
- let (_, value_type) = get_map_type(&mapfield.ty)?;
-
- let expanded = quote! {
- #[automatically_derived]
- impl #impl_generics NameAndVersionMap for #name #ty_generics #where_clause {
- type Value = #value_type;
-
- fn map_field(&self) -> &BTreeMap<NameAndVersion, Self::Value> {
- self.#mapfield_name.map_field()
- }
-
- fn map_field_mut(&mut self) -> &mut BTreeMap<NameAndVersion, Self::Value> {
- self.#mapfield_name.map_field_mut()
- }
-
- fn insert_or_error(&mut self, key: NameAndVersion, val: Self::Value) -> Result<(), name_and_version::Error> {
- self.#mapfield_name.insert_or_error(key, val)
- }
-
- fn num_crates(&self) -> usize {
- self.#mapfield_name.num_crates()
- }
-
- fn get_versions<'a, 'b>(&'a self, name: &'b str) -> Box<dyn Iterator<Item = (&'a NameAndVersion, &'a Self::Value)> + 'a> {
- self.#mapfield_name.get_versions(name)
- }
-
- fn get_versions_mut<'a, 'b>(&'a mut self, name: &'b str) -> Box<dyn Iterator<Item = (&'a NameAndVersion, &'a mut Self::Value)> + 'a> {
- self.#mapfield_name.get_versions_mut(name)
- }
-
- fn filter_versions<'a: 'b, 'b, F: Fn(&mut dyn Iterator<Item = (&'b NameAndVersion, &'b Self::Value)>,
- ) -> HashSet<Version> + 'a>(
- &'a self,
- f: F,
- ) -> Box<dyn Iterator<Item =(&'a NameAndVersion, &'a Self::Value)> + 'a> {
- self.#mapfield_name.filter_versions(f)
- }
- }
- };
-
- Ok(expanded)
- }
-
- fn get_struct(input: &DeriveInput) -> Result<&DataStruct> {
- match &input.data {
- Data::Struct(strukt) => Ok(strukt),
- _ => Err(Error::new_spanned(input, "Not a struct")),
- }
- }
-
- fn get_map_field(strukt: &DataStruct) -> Result<&Field> {
- for field in &strukt.fields {
- if let Ok((syn::Type::Path(path), _value_type)) = get_map_type(&field.ty) {
- if path.path.segments.len() == 1 && path.path.segments[0].ident == "NameAndVersion"
- {
- return Ok(field);
- }
- }
- }
- Err(Error::new_spanned(strukt.struct_token, "No field of type NameAndVersionMap"))
- }
-
- fn get_map_type(typ: &Type) -> Result<(&Type, &Type)> {
- if let syn::Type::Path(path) = &typ {
- if path.path.segments.len() == 1 && path.path.segments[0].ident == "BTreeMap" {
- if let PathArguments::AngleBracketed(args) = &path.path.segments[0].arguments {
- if args.args.len() == 2 {
- return Ok((get_type(&args.args[0])?, get_type(&args.args[1])?));
- }
- }
- }
- }
- Err(Error::new_spanned(typ, "Must be BTreeMap"))
- }
-
- fn get_type(arg: &GenericArgument) -> Result<&Type> {
- if let GenericArgument::Type(typ) = arg {
- return Ok(typ);
- }
- Err(Error::new_spanned(arg, "Could not extract argument type"))
- }
-}
