Python Examples of urllib.request.urlretrieve (2023)

def maybe_download(filename, url, destination_dir, expected_bytes = None, force = False): filepath = os.path.join(destination_dir, filename) if force or not os.path.exists(filepath): if not os.path.exists(destination_dir): os.makedirs(destination_dir) print('Attempting to download: ' + filename) filepath, _ = urlretrieve(url, filepath, reporthook = progress_bar) print('Download complete!') statinfo = os.stat(filepath) if expected_bytes != None: if statinfo.st_size == expected_bytes: print('Found and verified: ' + filename) else: raise Exception('Failed to verify: ' + filename + '. Can you get to it with a browser?') else: print('Found: ' + filename) print('The size of the file: ' + str(statinfo.st_size)) return filepath 

def _checkout(self, local_dir): user = self.config.get("user") repo = self.config.get("repo") version = self.config.get("version", "master") # TODO : Sanitize URL url = URL.format(user=user, repo=repo, version=version) logger.info("Downloading {}/{} from github".format(user, repo)) try: (filename, headers) = urllib.urlretrieve(url) except URLError as e: raise RuntimeError("Failed to download '{}'. '{}'".format(url, e.reason)) t = tarfile.open(filename) (cache_root, core) = os.path.split(local_dir) # Ugly hack to get the first part of the directory name of the extracted files tmp = t.getnames()[0] t.extractall(cache_root) os.rename(os.path.join(cache_root, tmp), os.path.join(cache_root, core)) 

def __init__(self, device, path_to_detector=None, verbose=False): super(SFDDetector, self).__init__(device, verbose) base_path = "face_alignment/ckpts" # Initialise the face detector if path_to_detector is None: path_to_detector = os.path.join( base_path, "s3fd_convert.pth") if not os.path.isfile(path_to_detector): print("Downloading the face detection CNN. Please wait...") path_to_temp_detector = os.path.join( base_path, "s3fd_convert.pth.download") if os.path.isfile(path_to_temp_detector): os.remove(os.path.join(path_to_temp_detector)) request_file.urlretrieve( "https://www.adrianbulat.com/downloads/python-fan/s3fd_convert.pth", os.path.join(path_to_temp_detector)) os.rename(os.path.join(path_to_temp_detector), os.path.join(path_to_detector)) self.face_detector = s3fd() self.face_detector.load_state_dict(torch.load(path_to_detector)) self.face_detector.to(device) self.face_detector.eval() 

def fetch_data(): """Download the data.""" train_file = tempfile.NamedTemporaryFile() test_file = tempfile.NamedTemporaryFile() req.urlretrieve("http://mlr.cs.umass.edu/ml/machine-learning-databases" "/adult/adult.data", train_file.name) req.urlretrieve("http://mlr.cs.umass.edu/ml/machine-learning-databases/" "adult/adult.test", test_file.name) df_train = pd.read_csv(train_file, names=COLUMNS, skipinitialspace=True) df_test = pd.read_csv(test_file, names=COLUMNS, skipinitialspace=True, skiprows=1) df_train[LABEL_COLUMN] = (df_train["income_bracket"] .apply(lambda x: ">50K" in x)).astype(int) df_test[LABEL_COLUMN] = (df_test["income_bracket"] .apply(lambda x: ">50K" in x)).astype(int) return df_train, df_test 

def loadData(src): print('Downloading ' + src) fname, h = urlretrieve(src, './delete.me') print('Done.') try: print('Extracting files...') with tarfile.open(fname) as tar: tar.extractall() print('Done.') print('Preparing train set...') trn = np.empty((0, numFeature + 1), dtype=np.int) for i in range(5): batchName = './cifar-10-batches-py/data_batch_{0}'.format(i + 1) trn = np.vstack((trn, readBatch(batchName))) print('Done.') print('Preparing test set...') tst = readBatch('./cifar-10-batches-py/test_batch') print('Done.') finally: os.remove(fname) return (trn, tst) 

def loadData(src, cimg): gzfname, h = urlretrieve(src, './delete.me') try: with gzip.open(gzfname) as gz: n = struct.unpack('I', gz.read(4)) if n[0] != 0x3080000: raise Exception('Invalid file: unexpected magic number.') n = struct.unpack('>I', gz.read(4))[0] if n != cimg: raise Exception('Invalid file: expected {0} entries.'.format(cimg)) crow = struct.unpack('>I', gz.read(4))[0] ccol = struct.unpack('>I', gz.read(4))[0] if crow != 28 or ccol != 28: raise Exception('Invalid file: expected 28 rows/cols per image.') res = np.fromstring(gz.read(cimg * crow * ccol), dtype=np.uint8) finally: os.remove(gzfname) return res.reshape((cimg, crow * ccol)) 

def load_or_download_mnist_files(filename, num_samples, local_data_dir): if (local_data_dir): local_path = os.path.join(local_data_dir, filename) else: local_path = os.path.join(os.getcwd(), filename) if os.path.exists(local_path): gzfname = local_path else: local_data_dir = os.path.dirname(local_path) if not os.path.exists(local_data_dir): os.makedirs(local_data_dir) filename = "http://yann.lecun.com/exdb/mnist/" + filename print ("Downloading from" + filename, end=" ") gzfname, h = urlretrieve(filename, local_path) print ("[Done]") return gzfname 

def _install_module(self, fullname): top, username, repo, modname = self._parse_fullname(fullname) url = 'https://raw.githubusercontent.com/%s/%s/master/%s' % (username, repo, modname+'.py') print('Downloading: ', url) try: tmp_file, resp = urlretrieve(url) with open(tmp_file, 'r') as f: new_content = f.read() if new_content=='Not Found': raise InstallError('remote file does not exist') except IOError: raise InstallError('error downloading file') new = tmp_file old = self._install_path(fullname) updated = self._update_if_changed(old, new) if updated=='updated': print('Updating module: ', fullname) elif updated=='installed': print('Installing module: ', fullname) elif updated=='noaction': print('Using existing version: ', fullname) 

def load_logos(filename): ''' Load logos from a geologos archive from <filename> <filename> can be either a local path or a remote URL. ''' if filename.startswith('http'): log.info('Downloading GeoLogos bundle: %s', filename) filename, _ = urlretrieve(filename, tmp.path('geologos.tar.xz')) log.info('Extracting GeoLogos bundle') with contextlib.closing(lzma.LZMAFile(filename)) as xz: with tarfile.open(fileobj=xz, encoding='utf8') as tar: tar.extractall(tmp.root, members=tar.getmembers()) log.info('Moving to the final location and cleaning up') if os.path.exists(logos.root): shutil.rmtree(logos.root) shutil.move(tmp.path('logos'), logos.root) log.info('Done') 

def import_shapers(logger): (_, zip_path) = tempfile.mkstemp() (_, http_message) = request.urlretrieve(url, zip_path) zip_file = ZipFile(zip_path) ex_dir = tempfile.mkdtemp() zip_file.extractall(ex_dir) shapefiles = glob.glob1(ex_dir, "*.shp") lm = LayerMapping(Parcel, "/data/shapefiles/M274TaxPar.shp", { "shape_leng": "SHAPE_Leng", "shape_area": "SHAPE_Area", "map_par_id": "MAP_PAR_ID", "loc_id": "LOC_ID", "poly_type": "POLY_TYPE", "map_no": "MAP_NO", "source": "SOURCE", "plan_id": "PLAN_ID", "last_edit": "LAST_EDIT", "town_id": "TOWN_ID", "shape": "POLYGON" }) 

def _load_cifar10(): def unflatten(images): return np.transpose(images.reshape((images.shape[0], 3, 32, 32)), [0, 2, 3, 1]) with tempfile.NamedTemporaryFile() as f: request.urlretrieve(URLS['cifar10'], f.name) tar = tarfile.open(fileobj=f) train_data_batches, train_data_labels = [], [] for batch in range(1, 6): data_dict = scipy.io.loadmat(tar.extractfile( 'cifar-10-batches-mat/data_batch_{}.mat'.format(batch))) train_data_batches.append(data_dict['data']) train_data_labels.append(data_dict['labels'].flatten()) train_set = {'images': np.concatenate(train_data_batches, axis=0), 'labels': np.concatenate(train_data_labels, axis=0)} data_dict = scipy.io.loadmat(tar.extractfile( 'cifar-10-batches-mat/test_batch.mat')) test_set = {'images': data_dict['data'], 'labels': data_dict['labels'].flatten()} train_set['images'] = _encode_png(unflatten(train_set['images'])) test_set['images'] = _encode_png(unflatten(test_set['images'])) return dict(train=train_set, test=test_set) 

def _load_cifar100(): def unflatten(images): return np.transpose(images.reshape((images.shape[0], 3, 32, 32)), [0, 2, 3, 1]) with tempfile.NamedTemporaryFile() as f: request.urlretrieve(URLS['cifar100'], f.name) tar = tarfile.open(fileobj=f) data_dict = scipy.io.loadmat(tar.extractfile('cifar-100-matlab/train.mat')) train_set = {'images': data_dict['data'], 'labels': data_dict['fine_labels'].flatten()} data_dict = scipy.io.loadmat(tar.extractfile('cifar-100-matlab/test.mat')) test_set = {'images': data_dict['data'], 'labels': data_dict['fine_labels'].flatten()} train_set['images'] = _encode_png(unflatten(train_set['images'])) test_set['images'] = _encode_png(unflatten(test_set['images'])) return dict(train=train_set, test=test_set)# Load a custom dataset from a local directory. 

def check_or_download_inception(inception_path): """ Checks if the path to the inception file is valid, or downloads the file if it is not present. """ INCEPTION_URL = 'http://download.tensorflow.org/models/image/imagenet/inception-2015-12-05.tgz' if inception_path is None: inception_path = '/tmp' inception_path = pathlib.Path(inception_path) model_file = inception_path / 'classify_image_graph_def.pb' if not model_file.exists(): print("Downloading Inception model") from urllib import request import tarfile fn, _ = request.urlretrieve(INCEPTION_URL) with tarfile.open(fn, mode='r') as f: f.extract('classify_image_graph_def.pb', str(model_file.parent)) return str(model_file) 

def maybe_download(filename, url, destination_dir, expected_bytes = None, force = False): filepath = os.path.join(destination_dir, filename) if force or not os.path.exists(filepath): if not os.path.exists(destination_dir): os.makedirs(destination_dir) print('Attempting to download: ' + filename) filepath, _ = urlretrieve(url, filepath, reporthook = progress_bar) print('Download complete!') statinfo = os.stat(filepath) if expected_bytes != None: if statinfo.st_size == expected_bytes: print('Found and verified: ' + filename) else: raise Exception('Failed to verify: ' + filename + '. Can you get to it with a browser?') else: print('Found: ' + filename) print('The size of the file: ' + str(statinfo.st_size)) return filepath 

def download_url(url, destination=None, progress_bar=True): def my_hook(t): last_b = [0] def inner(b=1, bsize=1, tsize=None): if tsize is not None: t.total = tsize if b > 0: t.update((b - last_b[0]) * bsize) last_b[0] = b return inner if progress_bar: with tqdm(unit='B', unit_scale=True, miniters=1, desc=url.split('/')[-1]) as t: filename, _ = urlretrieve(url, filename=destination, reporthook=my_hook(t)) else: filename, _ = urlretrieve(url, filename=destination) 

def run(self):while True:if self.joke_queue.empty() and self.page_queue.empty():print(self.name + '任务完成~')break# 2.从joke_queue队列中获取数据joke_info = self.joke_queue.get(timeout=40)username, content, img, alt, pubtime = joke_info# 3.上锁self.gLock.acquire()# 4.写入数据到csv中self.writer.writerow((username, content, img, alt, pubtime))# 5.下载图片到本地# file_name = alt + fileutils.get_file_suffix(img)# request.urlretrieve(img, './imgs/%s' % file_name)# 6.释放锁self.gLock.release()print('写入一条数据成功') 

def _regonize_captcha(self, image_url): """ 人工识别验证码【urllib+PIL】 :param image_url: :return: """ print('验证码的地址:%s,开始下载图片' % image_url) # 下载图片到本地 request.urlretrieve(image_url, 'captcha.png') print('下载图片完成，开始显示图片') # 显示在控制台，手动输入验证码 # 打开图片 image = Image.open('captcha.png') # 展示 image.show() # 提示输入验证码 captcha = input('请输入验证码:') return captcha 

def download_image(url, local_filename): if not download_images: return True try: urlretrieve(url, local_filename) return True except: return False# Download a given video via youtube-dl 

def download_or_copy_avatar(user, total_image_count, total_video_count, total_media_precount, year_str, month_str): # _orig existing means we already processed this user if 'profile_image_url_https_orig' in user: return False avatar_url = user['profile_image_url_https'] extension = os.path.splitext(avatar_url)[1] local_filename = "img/avatars/%s%s" % (user['screen_name'], extension) if not os.path.isfile(local_filename): can_be_copied = args.EARLIER_ARCHIVE_PATH and os.path.isfile(earlier_archive_path + local_filename) output_line("[%0.1f%%] %s/%s: %s avatar..." % ((total_image_count + total_video_count) / total_media_precount * 100, \ year_str, month_str, \ "Copying" if can_be_copied else "Downloading")) # If using an earlier archive as a starting point, try to see if the # avatar image is there and can be copied if can_be_copied: copyfile(earlier_archive_path + local_filename, local_filename) # Otherwise download it else: try: urlretrieve(avatar_url, local_filename) except: # Okay to quietly fail, this is just an avatar # (And, apparently, some avatars return 404.) return False user['profile_image_url_https_orig'] = user['profile_image_url_https'] user['profile_image_url_https'] = local_filename return True 

def task_8( img_url: str = 'https://i.imgur.com/B75zq0x.jpg') -> object: ''' Task 8: Module Args: img_url: address of an image Returns: result_img: an PIL Image Hints: * Make sure you have installed the PIL package * Take a look at utils.py first * You could easily find answers with Google ''' from urllib import request result_img = None # TODO: download the image from img_url with the request module # and add your student ID on it with draw_name() in the utils module # under src/. from PIL import Image import utils request.urlretrieve(img_url, 'test.jpg') result_img = Image.open('test.jpg') # You are allowed to change the img_url to your own image URL. utils.draw_text(result_img, "b07902115") # Display the image: # result_img.show() # Note: please comment this line when hand in. # If you are running on a server, use # result_img.save('test.jpg') # and copy the file to local or use Jupyter Notebook to render. # End of TODO return result_img 

def task_8( img_url: str = 'https://i.imgur.com/B75zq0x.jpg') -> object: ''' Task 8: Module Args: img_url: address of an image Returns: result_img: an PIL Image Hints: * Make sure you have installed the PIL package * Take a look at utils.py first * You could easily find answers with Google ''' # TODO: download the image from img_url with the request module # and add your student ID on it with draw_name() in the utils module # under src/. from urllib import request from PIL import Image, ImageFont, ImageDraw from utils import draw_text result_img = None local_filename, headers = request.urlretrieve(img_url) img = Image.open(local_filename) result_img = draw_text(img, 'B07902109') #result.img.show() # You are allowed to change the img_url to your own image URL. # Display the image: # result_img.show() # Note: please comment this line when hand in. # If you are running on a server, use # result.save('test.jpg') # and copy the file to local or use Jupyter Notebook to render. # End of TODO return result_img 

def task_8( img_url: str = 'https://i.imgur.com/B75zq0x.jpg') -> object: ''' Task 8: Module Args: img_url: address of an image Returns: result_img: an PIL Image Hints: * Make sure you have installed the PIL package * Take a look at utils.py first * You could easily find answers with Google ''' from urllib import request result_img = None # TODO: download the image from img_url with the request module # and add your student ID on it with draw_text() in the utils module # under src/. # You are allowed to change the img_url to your own image URL. # Display the image. If you are running on a server, change this line to # result.save('test.jpg') import utils request.urlretrieve(img_url, os.path.join(SRC_PATH, 'test.jpg')) result_img = utils.Image.open('test.jpg') utils.draw_text(result_img, 'b07902023') result_img.save('test.jpg') # result_img.show() # End of TODO return result_img 

def task_8( img_url: str = 'https://i.imgur.com/B75zq0x.jpg') -> object: from urllib import request from utils import draw_text from PIL import Image result_img = None local_filename, headers = request.urlretrieve(img_url) img = Image.open(local_filename) result_img = draw_text(img, "B07902001") ''' Task 8: Module Args: img_url: address of an image Returns: result_img: an PIL Image Hints: * Make sure you have installed the PIL package * Take a look at utils.py first * You could easily find answers with Google ''' # TODO: download the image from img_url with the request module # and add your student ID on it with draw_name() in the utils module # under src/. # You are allowed to change the img_url to your own image URL. # Display the image: # result_img.show() # Note: please comment this line when hand in. # If you are running on a server, use # result.save('test.jpg') # and copy the file to local or use Jupyter Notebook to render. # End of TODO return result_img 

def task_8( img_url: str = 'https://i.imgur.com/FmkNlkW.jpg') -> object: ''' Task 8: Module Args: img_url: address of an image Returns: result_img: an PIL Image Hints: * Make sure you have installed the PIL package * Take a look at utils.py first * You could easily find answers with Google ''' from urllib import request result_img = None # TODO: download the image from img_url with the request module # and add your student ID on it with draw_text() in the utils module # under src/. # You are allowed to change the img_url to your own image URL. # Display the image: # Note: please comment this line when hand in. # If you are running on a server, use # result.save('test.jpg') # and copy the file to local or use Jupyter Notebook to render. import utils request.urlretrieve(img_url, os.path.join(SRC_PATH, 'test.jpg')) result_img = utils.Image.open('test.jpg') utils.draw_text(result_img, 'B07902003') result_img.save('test.jpg') # result_img.show() # End of TODO return result_img 

def _checkout(self, local_dir): url = self.config.get("url") logger.info("Downloading...") user_agent = self.config.get("user-agent") if not self.config.get("verify_cert", True): import ssl ssl._create_default_https_context = ssl._create_unverified_context if user_agent and sys.version_info[0] >= 3: opener = urllib.build_opener() opener.addheaders = [("User-agent", user_agent)] urllib.install_opener(opener) try: (filename, headers) = urllib.urlretrieve(url) except (URLError, HTTPError) as e: raise RuntimeError("Failed to download '{}'. '{}'".format(url, e.reason)) filetype = self.config.get("filetype") if filetype == "tar": t = tarfile.open(filename) t.extractall(local_dir) elif filetype == "zip": with zipfile.ZipFile(filename, "r") as z: z.extractall(local_dir) elif filetype == "simple": _filename = url.rsplit("/", 1)[1] os.makedirs(local_dir) shutil.copy2(filename, os.path.join(local_dir, _filename)) else: raise RuntimeError( "Unknown file type '" + filetype + "' in [provider] section" ) 

def download_gitee(package, name, file_dir=None, project='logo'): """下载非结构文本数据 :param package: 包名 :param name: 文件名 :param file_dir: 保存文件的文件夹地址 :param project: 项目名 :return: 下载成功返回'success',失败返回'fail' """ if not file_dir: address = os.path.join(os.getcwd(), name) # 默认保存地址是运行项目当前的位置 elif os.path.isdir(file_dir): address = os.path.join(file_dir, name) elif not os.path.exists(file_dir): os.mkdir(file_dir) address = os.path.join(file_dir, name) else: address = file_dir url = GITEE.format(project=project, package=package, name=name) try: place = urlretrieve(url, address) # 下载 print('\033[1;33m' + place[0]) return 'success' except Exception as e: print(e) return 'fail' 

def _report_hook(count, block_size, total_size): """Display a progress bar for ``urlretrieve``""" progress_size = int(count * block_size) percent = min(100, int(count * block_size * 100 / total_size)) sys.stdout.write("\rDownloading %d/%d MB (%d%%)" % (progress_size / (1024 * 1024), total_size / (1024 * 1024), percent)) sys.stdout.flush() 

def fetch_url(url, file_path, progress_bar=_report_hook, remote_checksum=None): """Download a remote file Fetch a dataset pointed to by ``url``, check its SHA-256 checksum for integrity, and save it to ``file_path``. Parameters ---------- url : string URL of file to download file_path: string Path to the local file that will be created progress_bar : func callback, optional, default: built-in A callback to a function ``func(count, block_size, total_size)`` that will display a progress bar. remote_checksum : str, optional, default: None The expected SHA-256 checksum of the file. """ urlretrieve(url, file_path, progress_bar) checksum = _sha256(file_path) if remote_checksum != None and remote_checksum != checksum: raise IOError("{} has an SHA256 checksum ({}) " "differing from expected ({}), " "file may be corrupted.".format(file_path, checksum, remote_checksum)) 

def __init__(self, device, path_to_detector=None, verbose=False): super().__init__(device, verbose) base_path = os.path.join(appdata_dir('face_alignment'), "data") # Initialise the face detector if 'cuda' in device: if path_to_detector is None: path_to_detector = os.path.join( base_path, "mmod_human_face_detector.dat") if not os.path.isfile(path_to_detector): print("Downloading the face detection CNN. Please wait...") path_to_temp_detector = os.path.join( base_path, "mmod_human_face_detector.dat.download") if os.path.isfile(path_to_temp_detector): os.remove(os.path.join(path_to_temp_detector)) request_file.urlretrieve( "https://www.adrianbulat.com/downloads/dlib/mmod_human_face_detector.dat", os.path.join(path_to_temp_detector)) os.rename(os.path.join(path_to_temp_detector), os.path.join(path_to_detector)) self.face_detector = dlib.cnn_face_detection_model_v1(path_to_detector) else: self.face_detector = dlib.get_frontal_face_detector() 

def downloadFile(url): fname = url.split('/')[-1] urllib2.urlretrieve(url, fname, report)