🖥 🔍 Web Scraping with Python, Selenium, and Tor: A Powerful Combination

Веб-скрепинг с помощью Python, Selenium и Tor: Мощная комбинация для парсинга.

import subprocess
import time
from selenium import webdriver
from selenium.webdriver.common.proxy import Proxy, ProxyType
from selenium.webdriver.firefox.options import Options

# Define the command to start Tor. This will depend on your operating system and configuration.
command = '<<path_to_tor>>/tor.exe'

# Start the Tor process
tor_process = subprocess.Popen(command, stdout=subprocess.PIPE)

# Wait for Tor to start up
time.sleep(5)

# Sets up proxy settings for the Firefox browser driver
proxy_settings = Proxy({
'proxyType': ProxyType.MANUAL,
'socksProxy': '127.0.0.1:9050',
'socksVersion': 5
})

options = Options()
options.proxy = proxy_settings

driver = webdriver.Firefox(options=options)
driver.get('https://check.torproject.org') # Check if we are using Tor

# When you're done, don't forget to stop the Tor process
tor_process.terminate()

@pythonl

🦅 Trafilatura

Python & command-line tool to gather text on the Web: web crawling/scraping, extraction of text, metadata, comments

Это Python бибилиотека и инструмент командной строки для парсинга и сбора текста с сайтов.

Инструмент способен к сбору основного текста, метаданных и комментариев.

▪Github
▪Tutorials
▪Python Notebook

@pythonl

🤓 StackOverflow Question Scraper

Парсер вопросов StackOverflow на Python.

from bs4 import BeautifulSoup
import requests
import json

fmt = "https://stackoverflow.com/questions/tagged/{tag}?tab={filter}&pagesize=15"
filters = [
"1. Newest",
"2. Active",
"3. Bounties",
"4. Unanswered",
"5. Frequent",
"6. Votes",
]

tag = input("enter any question tag (python, java)\n")
print("\n".join(filters))
filter = int(input("enter the filter number (1, 3, 5)\n"))

try:
filter = filters[filter].split(" ")[-1]
except:
filter = "Votes"

# generate dynamic URL with user preferences
URL = fmt.format(tag=tag, filter=filter)

print("generated URL ", URL)
content = requests.get(URL).content

soup = BeautifulSoup(content, "lxml")

# return only question tags
def is_question(tag):
try:
return tag.get("id").startswith("question-summary-")
except:
return False


questions = soup.find_all(is_question)
question_data = []
if questions:
# extract question data like votes, title, link and date
for question in questions:
question_dict = {}
question_dict["votes"] = (
question.find(class_="s-post-summary--stats-item-number").get_text().strip()
)
h3 = question.find(class_="s-post-summary--content-title")
question_dict["title"] = h3.get_text().strip()
question_dict["link"] = "https://stackoverflow.com" + h3.find("a").get("href")
question_dict["date"] = (
question.find(class_="s-user-card--time").span.get_text().strip()
)
question_data.append(question_dict)

with open(f"questions-{tag}.json", "w") as f:
json.dump(question_data, f)

print("file exported")

else:
print(URL)
print("looks like there are no questions matching your tag ", tag)

@pythonl

⚡️Маст-хэв список программистов, каналы с последними книжными новинками, библиотеками, разбором кода и актуальной информацией, связанной с вашим языком программирования.
Лучший способ получать свежие обновлении и следить за трендами в разработке.

Python: t.me/pro_python_code
C#: t.me/csharp_ci
C/C++/ t.me/cpluspluc
Машинное обучение: t.me/ai_machinelearning_big_data
Data Science: t.me/data_analysis_ml
Devops: t.me/devOPSitsec
Go: t.me/Golang_google
Базы данных: t.me/sqlhub
Rust: t.me/rust_code
Javascript: t.me/javascriptv
React: t.me/react_tg
PHP: t.me/phpshka
Android: t.me/android_its
Мобильная разработка: t.me/mobdevelop
Linux: t.me/+A8jY79rcyKJlYWY6
Big Data: t.me/bigdatai
Хакинг: t.me/linuxkalii
Java: t.me/javatg

💼 Папка с вакансиями: t.me/addlist/_zyy_jQ_QUsyM2Vi
Папка Go разработчика: t.me/addlist/MUtJEeJSxeY2YTFi
Папка Python разработчика: t.me/addlist/eEPya-HF6mkxMGIy

🎞 YouTube канал: https://www.youtube.com/@uproger

😆ИТ-Мемы: t.me/memes_prog

🇬🇧Английский: t.me/english_forprogrammers

📶 Get Saved Wi-Fi Passwords

Скрипт для извлечения всех сохраненных wi-fi паролей в текстовый файл.

import subprocess
import re

# Get all the Wi-Fi profiles (ssid)
out = subprocess.check_output("netsh wlan show profiles").decode()

# Filter out only profile names from the output
matches = re.findall(r"(All User Profile)(.*)", out)

# List comprehension to remove any \n \r \t and spaces
profiles = [str(match[1]).split(":")[1].strip() for match in matches]


# File object to store passwords with ssid
with open("passwords.txt", "w+") as f:

# Traversing each profile
for profile in profiles:
# try/except block to keep the script from crashing if there was an error while execution
try:
# Get password using key=clear flag
get_pass = subprocess.check_output(
f'netsh wlan show profile "{profile}" key=clear'
).decode()

# Filter out the Password line from the output
pass_by_profile = re.search(r"(Key Content)(.*)", get_pass)

# Check if the password is present or wi-fi was open
if pass_by_profile:
password = pass_by_profile.group().split(":")[1].strip()
else:
password = "THE WIFI IS OPEN"

# Write the profile name and password to the text file
f.write(f"{profile} : {password}\n")
except Exception:
continue

@pythonl