v2 of the paper

fra31 · Dec 14, 2020 · 5539786 · 5539786
1 parent 4d4e7f9
commit 5539786
Show file tree

Hide file tree

Showing 3 changed files with 513 additions and 127 deletions.
diff --git a/README.md b/README.md
@@ -36,7 36,7 @@ Moreover, for L0-perturbations **Sparse-RS** can even outperform strong white-bo
 
 
 ## Code of Sparse-RS
-The code is tested under Python 3.6 and PyTorch 1.4.0. It automatically downloads the pretrained models (either VGG-16-BN or ResNet-50) and requires access to ImageNet validation set.
 The code is tested under Python 3.8.5 and PyTorch 1.8.0. It automatically downloads the pretrained models (either VGG-16-BN or ResNet-50) and requires access to ImageNet validation set.
 
 The following are examples of how to run the attacks in the different threat models.
 
@@ -51,30 51,44 @@ and for targeted attacks please use `--targeted --n_queries=100000 --alpha_init=
 
 As additional options the flag `--constant_schedule` uses a constant schedule for `alpha` instead of the piecewise constant decreasing one, while with `--seed=N` it is possible to set a custom random seed.
 
-### Image-specific patches
-For image- and location-specific patches of size 30x30 (with `k=900`)
 ### Image-specific patches and frames
 For image- and location-specific patches of size 20x20 (with `k=400`)
 ```
 CUDA_VISIBLE_DEVICES=0 python eval.py --norm=patches \
- --model=[pt_vgg | pt_resnet] --n_queries=10000 --alpha_init=0.3 \
- --data_path=/path/to/validation/set --k=900 --n_ex=100
  --model=[pt_vgg | pt_resnet] --n_queries=10000 --alpha_init=0.4 \
  --data_path=/path/to/validation/set --k=400 --n_ex=100
 ```
 
 For targeted patches (size 40x40) please use `--targeted --n_queries=50000 --alpha_init=0.1 --k=1600`. The target class is randomly chosen for each point.
 
 while for image-specific frames of width 2 pixels (with `k=2`)
 ```
 CUDA_VISIBLE_DEVICES=0 python eval.py --norm=frames \
  --model=[pt_vgg | pt_resnet] --n_queries=10000 --alpha_init=0.5 \
  --data_path=/path/to/validation/set --k=2 --n_ex=100
 ```
 
 For targeted frames (width of 3 pixels) please use `--targeted --n_queries=50000 --alpha_init=0.5 --k=3`. The target class is randomly chosen for each point
 
 ### Universal patches and frames
-For universal untargeted patches of size 50x50 (with `k=2500`)
 For universal targeted patches of size 50x50 (with `k=2500`)
 ```
 CUDA_VISIBLE_DEVICES=0 python eval.py \
  --norm=patches_universal --model=[pt_vgg | pt_resnet] \
  --n_queries=100000 --alpha_init=0.3 \
- --data_path=/path/to/validation/set --k=2500 --n_ex=100
  --data_path=/path/to/validation/set --k=2500 \
  --n_ex=30 --targeted --target_class=530
 ```
-while for universal untargeted frames of width 4 (with `k=4`)
 
 and for universal frames of width 6 pixels (`k=6`)
 ```
 CUDA_VISIBLE_DEVICES=0 python eval.py \
  --norm=frames_universal --model=[pt_vgg | pt_resnet] \
- --n_queries=100000 --alpha_init=0.005 \
- --data_path=/path/to/validation/set --k=4 --n_ex=100
  --n_queries=100000 --alpha_init=1.667 \
  --data_path=/path/to/validation/set --k=6 \
  --n_ex=30 --targeted --target_class=530
 ```
-For **universal targeted** attacks add at the previous commands `--targeted --target_class=920` with the number corresponding to the target label.
 The argument `--target_class` specifies the number corresponding to the target label. To generate universal attacks we use batches of 30 images resampled every 10000 queries.
 
 ## Visualizing resulting images
 We provide a script `vis_images.py` to visualize the images produced by the attacks. To use it please run

diff --git a/eval.py b/eval.py
@@ -8,16 8,18 @@
 import torchvision.transforms as transforms
 from torchvision import models as torch_models
 
 import sys
 import time
 from datetime import datetime
 
 model_class_dict = {'pt_vgg': torch_models.vgg16_bn,
  'pt_resnet': torch_models.resnet50,
  }
 
-
 class PretrainedModel():
  def __init__(self, modelname):
  model_pt = model_class_dict[modelname](pretrained=True)
  #model.eval()
  self.model = nn.DataParallel(model_pt.cuda())
  self.model.eval()
  self.mu = torch.Tensor([0.485, 0.456, 0.406]).float().view(1, 3, 1, 1).cuda()
@@ -34,7 36,6 @@ def forward(self, x):
  def __call__(self, x):
  return self.predict(x)
 
-
 def random_target_classes(y_pred, n_classes):
  y = torch.zeros_like(y_pred)
  for counter in range(y_pred.shape[0]):
@@ -45,8 46,8 @@ def random_target_classes(y_pred, n_classes):
 
  return y.long()
 
-
 if __name__ == '__main__':
 
  parser = argparse.ArgumentParser()
 
  parser.add_argument('--dataset', type=str, default='ImageNet')
@@ -57,21 58,29 @@ def random_target_classes(y_pred, n_classes):
  parser.add_argument('--loss', type=str, default='margin')
  parser.add_argument('--model', default='pt_vgg', type=str)
  parser.add_argument('--n_ex', type=int, default=1000)
- parser.add_argument('--attack', type=str, default='sparse-rs')
  parser.add_argument('--attack', type=str, default='rs_attack')
  parser.add_argument('--n_queries', type=int, default=1000)
  parser.add_argument('--targeted', action='store_true')
  parser.add_argument('--target_class', type=int)
  parser.add_argument('--seed', type=int, default=0)
  parser.add_argument('--constant_schedule', action='store_true')
  parser.add_argument('--save_dir', type=str, default='./results')
 
  # Sparse-RS parameter
  parser.add_argument('--alpha_init', type=float, default=.3)
  parser.add_argument('--resample_period_univ', type=int)
  parser.add_argument('--loc_update_period', type=int)
 
  args = parser.parse_args()
 
  if args.data_path is None:
- args.data_path = "/scratch/maksym/imagenet/val_orig"
  args.data_path = "/home/scratch/datasets/imagenet/val"
 
  args.eps = args.k   0
  args.bs = args.n_ex   0
  args.p_init = args.alpha_init   0.
  args.resample_loc = args.resample_period_univ
  args.update_loc_period = args.loc_update_period
 
  if args.dataset == 'ImageNet':
  # load pretrained model
@@ -95,25 104,30 @@ def random_target_classes(y_pred, n_classes):
  testiter = iter(test_loader)
  x_test, y_test = next(testiter)
 
- if args.attack in ['sparse-rs']:
  if args.attack in ['rs_attack']:
  # run Sparse-RS attacks
- if not os.path.exists('./results/{}/'.format(args.dataset)):
- os.makedirs('./results/{}/'.format(args.dataset))
- if not os.path.exists('./results/logs/'):
- os.makedirs('./results/logs/')
  logsdir = '{}/logs_{}_{}'.format(args.save_dir, args.attack, args.norm)
  savedir = '{}/{}_{}'.format(args.save_dir, args.attack, args.norm)
  if not os.path.exists(savedir):
  os.makedirs(savedir)
  if not os.path.exists(logsdir):
  os.makedirs(logsdir)
 
  if args.targeted or 'universal' in args.norm:
  args.loss = 'ce'
- param_run = '{}_{}_{}_1_{}_nqueries_{:.0f}_alphainit_{:.2f}_loss_{}_eps_{:.0f}_targeted_{}_targetclass_{}_seed_{:.0f}'.format(
- args.attack, args.norm, args.model, args.n_ex, args.n_queries, args.alpha_init,
  data_loader = testiter if 'universal' in args.norm else None
 
  param_run = '{}_{}_{}_1_{}_nqueries_{:.0f}_pinit_{:.2f}_loss_{}_eps_{:.0f}_targeted_{}_targetclass_{}_seed_{:.0f}'.format(
  args.attack, args.norm, args.model, args.n_ex, args.n_queries, args.p_init,
  args.loss, args.eps, args.targeted, args.target_class, args.seed)
  if args.constant_schedule:
  param_run  = '_constantpinit'
 
  from rs_attacks import RSAttack
  adversary = RSAttack(model, norm=args.norm, eps=int(args.eps), verbose=True, n_queries=args.n_queries,
- alpha_init=args.alpha_init, log_path='./results/{}/log_run_{}_{}.txt'.format('logs', str(datetime.now())[:-7], param_run),
- loss=args.loss, targeted=args.targeted, seed=args.seed, constant_schedule=args.constant_schedule)
  p_init=args.p_init, log_path='{}/log_run_{}_{}.txt'.format(logsdir, str(datetime.now())[:-7], param_run),
  loss=args.loss, targeted=args.targeted, seed=args.seed, constant_schedule=args.constant_schedule,
  data_loader=data_loader, resample_loc=args.resample_loc)
 
  # set target classes
  if args.targeted and 'universal' in args.norm:
@@ -127,7 141,7 @@ def random_target_classes(y_pred, n_classes):
  y_test = random_target_classes(y_test, 1000)
  print('target labels', y_test)
 
- bs = min(args.bs, 250)
  bs = min(args.bs, 500)
  assert args.n_ex % args.bs == 0
  adv_complete = x_test.clone()
  qr_complete = torch.zeros([x_test.shape[0]]).cpu()
@@ -187,21 201,22 @@ def random_target_classes(y_pred, n_classes):
  adversary.logger.log('max L0 perturbation {:.0f} - nan in img {} - max img {:.5f} - min img {:.5f}'.format(
  res.max(), (adv_complete != adv_complete).sum(), adv_complete.max(), adv_complete.min()))
 
- ind_succ = (((pred_adv == 0.) * (pred == 1.)) == 1.).nonzero().squeeze()
- adversary.logger.log('success rate={:.0f}/{:.0f} ({:.2%}) - avg # queries {:.1f} - med # queries {:.1f}'.format(
- pred.sum() - pred_adv.sum(), pred.sum(), (pred.sum() - pred_adv.sum()).float() / pred.sum(),
- qr_complete[ind_succ].float().mean(), torch.median(qr_complete[ind_succ].float())))
  ind_corrcl = pred == 1.
  ind_succ = (pred_adv == 0.) * (pred == 1.)
 
  str_stats = 'success rate={:.0f}/{:.0f} ({:.2%}) \n'.format(
  pred.sum() - pred_adv.sum(), pred.sum(), (pred.sum() - pred_adv.sum()).float() / pred.sum())  \
  '[successful points] avg # queries {:.1f} - med # queries {:.1f}\n'.format(
  qr_complete[ind_succ].float().mean(), torch.median(qr_complete[ind_succ].float()))
  qr_complete[~ind_succ] = args.n_queries   0
  str_stats  = '[correctly classified points] avg # queries {:.1f} - med # queries {:.1f}\n'.format(
  qr_complete[ind_corrcl].float().mean(), torch.median(qr_complete[ind_corrcl].float()))
  adversary.logger.log(str_stats)
 
  # save results depending on the threat model
  if args.norm in ['L0', 'patches', 'frames']:
- save_path = './results/{}/{}_{}_{}_1_{}_nqueries_{:.0f}_alphainit_{:.2f}_loss_{}_eps_{:.0f}_targeted_{}_seed_{:.0f}'.format(
- args.dataset, args.attack, args.norm, args.model, args.n_ex, args.n_queries, args.alpha_init,
- args.loss, args.eps, args.targeted, args.seed)
- if args.constant_schedule:
- save_path  = '_constantschedule'
-
- torch.save({'adv': adv_complete, 'qr': qr_complete}, '{}.pth'.format(save_path))
  torch.save({'adv': adv_complete, 'qr': qr_complete},
  '{}/{}.pth'.format(savedir, param_run))
 
  elif args.norm in ['patches_universal']:
  # extract and save patch
@@ -213,9 228,7 @@ def random_target_classes(y_pred, n_classes):
  patch = adv_complete[ind, :, loc[0]:loc[0]   s, loc[1]:loc[1]   s].unsqueeze(0)
 
  torch.save({'adv': adv_complete, 'patch': patch},
- './results/{}/{}_{}_{}_1_{}_nqueries_{:.0f}_alphainit_{:.2f}_loss_{}_eps_{:.0f}_targeted_{}.pth'.format(
- args.dataset, args.attack, args.norm, args.model, args.n_ex, args.n_queries,
- args.alpha_init, args.loss, args.eps, args.targeted))
  '{}/{}.pth'.format(savedir, param_run))
 
  elif args.norm in ['frames_universal']:
  # extract and save frame and indeces of the perturbed pixels
@@ -231,10 244,5 @@ def random_target_classes(y_pred, n_classes):
  frame = adv_complete[ind_img, :, ind[:, 0], ind[:, 1]]
 
  torch.save({'adv': adv_complete, 'frame': frame, 'ind': ind},
- './results/{}/{}_{}_{}_1_{}_nqueries_{:.0f}_alphainit_{:.3f}_loss_{}_eps_{:.0f}_targeted_{}_targetclass_{}.pth'.format(
- args.dataset, args.attack, args.norm, args.model, args.n_ex, args.n_queries,
- args.alpha_init, args.loss, args.eps, args.targeted, args.target_class))
  '{}/{}.pth'.format(savedir, param_run))
 
- else:
- raise ValueError('unknown attack')
-