提交 5fe26266 编辑于 作者: Galen Andrew's avatar Galen Andrew 提交者: tensorflow-copybara
浏览文件

Remove clipping_compositions.py. These were intended to be a "mid-level" API...

Remove clipping_compositions.py. These were intended to be a "mid-level" API linking the low level aggregators with useful high-level compositions, but it is cleaner to skip this step.

PiperOrigin-RevId: 343392858
上级 bf7cd4b1
......@@ -27,35 +27,6 @@ py_library(
],
)
py_library(
name = "clipping_compositions",
srcs = ["clipping_compositions.py"],
srcs_version = "PY3",
deps = [
":clipping_factory",
":factory",
":mean_factory",
":quantile_estimation",
"//tensorflow_federated/python/core/api:computation_types",
"//tensorflow_federated/python/core/api:computations",
"//tensorflow_federated/python/core/api:intrinsics",
],
)
py_test(
name = "clipping_compositions_test",
srcs = ["clipping_compositions_test.py"],
python_version = "PY3",
srcs_version = "PY3",
deps = [
":clipping_compositions",
":factory",
"//tensorflow_federated/python/core/api:computation_types",
"//tensorflow_federated/python/core/api:test_case",
"//tensorflow_federated/python/core/backends/native:execution_contexts",
],
)
py_library(
name = "clipping_factory",
srcs = ["clipping_factory.py"],
......
# Copyright 2020, The TensorFlow Federated Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Functions providing recommended aggregator compositions."""
import tensorflow as tf
import tensorflow_privacy
from tensorflow_federated.python.aggregators import clipping_factory
from tensorflow_federated.python.aggregators import factory
from tensorflow_federated.python.aggregators import mean_factory
from tensorflow_federated.python.aggregators import quantile_estimation
from tensorflow_federated.python.core.api import computation_types
from tensorflow_federated.python.core.api import computations
from tensorflow_federated.python.core.api import intrinsics
def _affine_transform(multiplier, increment):
transform_tf_comp = computations.tf_computation(
lambda value: multiplier * value + increment, tf.float32)
return computations.federated_computation(
lambda value: intrinsics.federated_map(transform_tf_comp, value),
computation_types.at_server(tf.float32))
def _make_quantile_estimation_process(initial_estimate: float,
target_quantile: float,
learning_rate: float):
return quantile_estimation.PrivateQuantileEstimationProcess(
tensorflow_privacy.NoPrivacyQuantileEstimatorQuery(
initial_estimate=initial_estimate,
target_quantile=target_quantile,
learning_rate=learning_rate,
geometric_update=True))
def adaptive_zeroing_mean(
initial_quantile_estimate: float,
target_quantile: float,
multiplier: float,
increment: float,
learning_rate: float,
norm_order: bool,
no_nan_mean: bool = False) -> factory.WeightedAggregationFactory:
"""Creates a factory for mean with adaptive zeroing.
Estimates value at quantile `Z` of value norm distribution and zeroes out
values whose norm is greater than `rZ + i` for multiplier `r` and increment
`i`. The quantile `Z` is estimated using the geometric method described in
Thakkar et al. 2019, "Differentially Private Learning with Adaptive Clipping"
(https://arxiv.org/abs/1905.03871) without noise added (so not differentially
private).
Args:
initial_quantile_estimate: The initial estimate of the target quantile `Z`.
target_quantile: Which quantile to match, as a float in [0, 1]. For example,
0.5 for median, or 0.98 to zero out only the largest 2% of updates (if
multiplier=1 and increment=0).
multiplier: Factor `r` in zeroing norm formula `rZ + i`.
increment: Increment `i` in zeroing norm formula `rZ + i`.
learning_rate: Learning rate for quantile matching algorithm.
norm_order: A float for the order of the norm. Must be 1, 2, or np.inf.
no_nan_mean: A bool. If True, the computed mean is 0 if sum of weights is
equal to 0.
Returns:
A factory that performs mean after adaptive clipping.
"""
zeroing_quantile = _make_quantile_estimation_process(
initial_estimate=initial_quantile_estimate,
target_quantile=target_quantile,
learning_rate=learning_rate)
zeroing_norm = zeroing_quantile.map(_affine_transform(multiplier, increment))
mean = mean_factory.MeanFactory(no_nan_division=no_nan_mean)
return clipping_factory.ZeroingFactory(zeroing_norm, mean, norm_order)
def adaptive_zeroing_clipping_mean(
initial_zeroing_quantile_estimate: float,
target_zeroing_quantile: float,
zeroing_multiplier: float,
zeroing_increment: float,
zeroing_learning_rate: float,
zeroing_norm_order: bool,
initial_clipping_quantile_estimate: float,
target_clipping_quantile: float,
clipping_learning_rate: float,
no_nan_mean: bool = False) -> factory.WeightedAggregationFactory:
"""Makes a factory for mean with adaptive zeroing and clipping.
Estimates value at quantile `Z` of value norm distribution and zeroes out
values whose norm is greater than `rZ + i` for multiplier `r` and increment
`i`. Also estimates value at quantile `C` and clips values whose L2 norm is
greater than `C` (without any multiplier or increment). The quantiles are
estimated using the geometric method described in Thakkar et al. 2019,
"Differentially Private Learning with Adaptive Clipping"
(https://arxiv.org/abs/1905.03871) without noise added (so not differentially
private). Zeroing occurs before clipping, so the estimation process for `C`
uses already zeroed values.
Note while the zeroing_norm_order may be 1.0 or np.inf, only L2 norm is used
for clipping.
Args:
initial_zeroing_quantile_estimate: The initial estimate of the target
quantile `Z` for zeroing.
target_zeroing_quantile: Which quantile to match for zeroing, as a float in
[0, 1]. For example, 0.5 for median, or 0.98 to zero out only the largest
2% of updates (if multiplier=1 and increment=0).
zeroing_multiplier: Factor `r` in zeroing norm formula `rZ + i`.
zeroing_increment: Increment `i` in zeroing norm formula `rZ + i`.
zeroing_learning_rate: Learning rate for zeroing quantile estimate.
zeroing_norm_order: A float for the order of the norm for zeroing. Must be
1, 2, or np.inf.
initial_clipping_quantile_estimate: The initial estimate of the target
quantile `C` for clipping. (Multiplier and increment are not used for
clipping.)
target_clipping_quantile: Which quantile to match for clipping, as a float
in [0, 1].
clipping_learning_rate: Learning rate for clipping quantile estimate.
no_nan_mean: A bool. If True, the computed mean is 0 if sum of weights is
equal to 0.
Returns:
A factory that performs mean after adaptive zeroing and clipping.
"""
zeroing_quantile = _make_quantile_estimation_process(
initial_estimate=initial_zeroing_quantile_estimate,
target_quantile=target_zeroing_quantile,
learning_rate=zeroing_learning_rate)
zeroing_norm = zeroing_quantile.map(
_affine_transform(zeroing_multiplier, zeroing_increment))
clipping_norm = _make_quantile_estimation_process(
initial_estimate=initial_clipping_quantile_estimate,
target_quantile=target_clipping_quantile,
learning_rate=clipping_learning_rate)
mean = mean_factory.MeanFactory(no_nan_division=no_nan_mean)
clip = clipping_factory.ClippingFactory(clipping_norm, mean)
return clipping_factory.ZeroingFactory(zeroing_norm, clip, zeroing_norm_order)
# Copyright 2020, The TensorFlow Federated Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Tests for factory compositions."""
import numpy as np
import tensorflow as tf
from tensorflow_federated.python.aggregators import clipping_compositions as compositions
from tensorflow_federated.python.aggregators import factory
from tensorflow_federated.python.core.api import computation_types
from tensorflow_federated.python.core.api import test_case
from tensorflow_federated.python.core.backends.native import execution_contexts
class ClippingCompositionsTest(test_case.TestCase):
def test_adaptive_zeroing_mean(self):
factory_ = compositions.adaptive_zeroing_mean(
initial_quantile_estimate=1.0,
target_quantile=0.5,
multiplier=2.0,
increment=1.0,
learning_rate=np.log(4.0),
norm_order=np.inf)
self.assertIsInstance(factory_, factory.WeightedAggregationFactory)
process = factory_.create_weighted(
value_type=computation_types.to_type(tf.float32),
weight_type=computation_types.to_type(tf.float32))
state = process.initialize()
# Quantile estimate is 1.0, zeroing norm is 3.0.
client_data = [1.5, 3.5]
client_weight = [1.0, 1.0]
output = process.next(state, client_data, client_weight)
self.assertAllClose(1.5 / 2.0, output.result)
self.assertAllClose(3.0, output.measurements['zeroing_norm'])
self.assertAllClose(1.0, output.measurements['zeroed_count'])
# New quantile estimate is 1 * exp(0.5 ln(4)) = 2, zeroing norm is 5.0.
output = process.next(output.state, client_data, client_weight)
self.assertAllClose(5.0 / 2.0, output.result)
self.assertAllClose(5.0, output.measurements['zeroing_norm'])
self.assertAllClose(0.0, output.measurements['zeroed_count'])
def test_adaptive_zeroing_clipping_mean(self):
factory_ = compositions.adaptive_zeroing_clipping_mean(
initial_zeroing_quantile_estimate=1.0,
target_zeroing_quantile=0.5,
zeroing_multiplier=2.0,
zeroing_increment=2.0,
zeroing_learning_rate=np.log(4.0),
zeroing_norm_order=np.inf,
initial_clipping_quantile_estimate=2.0,
target_clipping_quantile=0.0,
clipping_learning_rate=np.log(4.0))
self.assertIsInstance(factory_, factory.WeightedAggregationFactory)
process = factory_.create_weighted(
value_type=computation_types.to_type(tf.float32),
weight_type=computation_types.to_type(tf.float32))
state = process.initialize()
client_data = [3.0, 4.5]
client_weight = [1.0, 1.0]
# Zero quantile: 1.0, zero norm: 4.0, clip quantile (norm): 2.0.
output = process.next(state, client_data, client_weight)
self.assertAllClose(2.0 / 2.0, output.result)
self.assertAllClose(4.0, output.measurements['zeroing_norm'])
self.assertAllClose(1.0, output.measurements['zeroed_count'])
clip_measurements = output.measurements['agg_process']
self.assertAllClose(2.0, clip_measurements['clipping_norm'])
self.assertAllClose(1.0, clip_measurements['clipped_count'])
# New zero quantile: 1 * exp(0.5 ln(4)) = 2
# New zero norm is 6.0
# New clip quantile (norm) is 2 * exp(-0.5 ln(4)) = 1
output = process.next(output.state, client_data, client_weight)
self.assertAllClose(2.0 / 2.0, output.result)
self.assertAllClose(6.0, output.measurements['zeroing_norm'])
self.assertAllClose(0.0, output.measurements['zeroed_count'])
clip_measurements = output.measurements['agg_process']
self.assertAllClose(1.0, clip_measurements['clipping_norm'])
self.assertAllClose(2.0, clip_measurements['clipped_count'])
if __name__ == '__main__':
execution_contexts.set_local_execution_context()
test_case.main()
支持 Markdown
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册