Merge pull request #1137 from ct-Open-Source/development

Release v2.4.6

.env-template

@@ -0,0 +1,6 @@
+#Attention: "host" means your computer where you are running docker on 
+
+WLAN=wlan0                          #must match the name of your wlan-interface on your host, you may find it with ifconfig 
+AP=vtrust-flash                     #the name of the created AP, can be anything you want
+GATEWAY=10.42.42.1                  #gateway address, leave it here
+LOCALBACKUPDIR=./data/backups       #location on your host where you want to store backuos of the old firmware & logs

.gitignore

@@ -109,8 +109,9 @@ node_modules/
 device-info.txt
 scripts/smartconfig/package-lock.json
 
-# firmware backups directory
+# firmware backups directory and data-file
 backups/
+data
 
 # flag files
 eula_accepted

Dockerfile

@@ -1,13 +1,13 @@
-FROM phusion/baseimage:master
+FROM alpine:3.13
 
-RUN apt-get update && apt-get install -y sudo iproute2 iputils-ping
+RUN apk add --update bash git iw dnsmasq hostapd screen curl py3-pip py3-wheel python3-dev mosquitto haveged net-tools openssl openssl-dev gcc musl-dev linux-headers sudo coreutils grep iproute2 ncurses
 
-RUN echo '* libraries/restart-without-asking boolean true' | sudo debconf-set-selections
+RUN python3 -m pip install --upgrade paho-mqtt tornado git+https://github.com/drbild/sslpsk.git pycryptodomex
 
 COPY docker/bin /usr/bin/
 
 COPY . /usr/bin/tuya-convert
 
-RUN cd /usr/bin/tuya-convert && ./install_prereq.sh
+WORKDIR "/usr/bin/tuya-convert"
 
-RUN mkdir -p /etc/service/tuya && cd /etc/service/tuya && ln -s /usr/bin/config.sh run
+ENTRYPOINT ["tuya-start"]

README.md

@@ -33,13 +33,18 @@ These scripts were tested in
 * a Raspberry Pi 3B / 3B+ / 4B with Raspbian Stretch and its internal Wifi chip
 * a Raspberry Pi 3B / 3B+ Raspberry Pi OS Buster (previously called Raspbian) and its internal Wifi chip
 * a Raspberry Pi 3B+ + USB-WIFI with an image from [here](https://www.offensive-security.com/kali-linux-arm-images/)
-* Ubuntu 18.04.3 64Bit in VirtualBox on Win10 with a [cheap RTL8188CU Wifi Adapter](http://s.click.aliexpress.com/e/KrKIoPdI) connected to the VM
+* Ubuntu 18.04.3 64Bit in VirtualBox on Win10 with a cheap RTL8188CU Wifi Adapter connected to the VM
+* Ubuntu Mate 18.04.5 32bit in VirtualBox on Win 10 with cheap Ralink 802.11n WLAN (MediaTek RT5370) WiFi Adapter connected to VM
 
 Any Linux with a Wifi adapter which can act as an Access Point should also work. Please note that we have tested the Raspberry Pi with clean installations only. If you use your Raspberry Pi for anything else, we recommend using another SD card with a clean installation.
 
 ## PROCEDURE
 
-On January 28th, 2019, Tuya started [distributing a patch](https://www.heise.de/newsticker/meldung/Smart-Home-Hack-Tuya-veroeffentlicht-Sicherheitsupdate-4292028.html) that prevented older versions of tuya-convert from completing successfully. We have since developed a work around to enable OTA flashing once again, but there is always the possibility that Tuya will respond with yet another patch. To ensure the best chance of success, **do not connect your device with the official app** as it may automatically update the device, preventing you from flashing with tuya-convert. It is up to the individual brands to update their firmware, so some devices may be affected sooner than others.
+On January 28th, 2019, Tuya started [distributing a patch](https://www.heise.de/newsticker/meldung/Smart-Home-Hack-Tuya-veroeffentlicht-Sicherheitsupdate-4292028.html) that prevented older versions of tuya-convert from completing successfully. We have since developed a work around to enable OTA flashing on some newer devices, but Tuya has since released [yet another patch](https://github.com/ct-Open-Source/tuya-convert/issues/483).
+
+To ensure the best chance of success, **do not connect your device with the official app** as it may automatically update the device, preventing you from flashing with tuya-convert. It is up to the individual brands to update their firmware, so some devices may be affected sooner than others.
+
+Unfortunately many devices have already been shipping with the new patched firmware, evident by a PSK ID beginning with 02 in smarthack-psk.log. There is no workaround at this time. Additionally, manufacturers have been silently switching from the ESP82xx based modules to other chipsets, making it impossible to install alternative ESP firmware on these devices.
 
 ### INSTALLATION
     # git clone https://github.com/ct-Open-Source/tuya-convert
@@ -50,7 +55,7 @@ On January 28th, 2019, Tuya started [distributing a patch](https://www.heise.de/
 BE SURE THE FIRMWARE FITS YOUR DEVICE!
 1. Place your binary file in the `/files/` directory or use one of the included firmware images.
 
-	Currently a [Tasmota](https://github.com/arendst/Tasmota) `tasmota-wifiman.bin` build is included in the Tuya-Convert package. You can easily update to the [current maintenance release](http://thehackbox.org/tasmota) via OTA **after** the flashing process completes successfully. The included binary does not have any specific hardware configured. Once flashed using Tuya-Convert you will need to configure your device(s) properly. Please note that while we include this firmware for your convenience, we are not affiliated with the Tasmota project and cannot provide support for post installation issues. Please refer to the [Tasmota project](https://github.com/arendst/Tasmota) and [its documentation](http://tasmota.com) for configuration and support. **IMPORTANT:** If you still want to update the tasmota binary before using Tuya-Convert, always pickup `tasmota-wifiman.bin`. Never ever use `tasmota-minimal.bin` as you will brick your device.
+	Currently a [Tasmota](https://github.com/arendst/Tasmota) `tasmota-lite.bin` build is included in the Tuya-Convert package. You can easily update to the [current maintenance release](http://thehackbox.org/tasmota) via OTA **after** the flashing process completes successfully. The included binary does not have any specific hardware configured. Once flashed using Tuya-Convert you will need to configure your device(s) properly. Please note that while we include this firmware for your convenience, we are not affiliated with the Tasmota project and cannot provide support for post installation issues. Please refer to the [Tasmota project](https://github.com/arendst/Tasmota) and [its documentation](http://tasmota.com) for configuration and support. **IMPORTANT:** If you still want to update the tasmota binary before using Tuya-Convert, always pickup `tasmota-lite.bin`. **Never ever** use `tasmota-minimal.bin` as you will brick your device.
 
 	An ESPurna [1.13.5](https://github.com/xoseperez/espurna/releases/tag/1.13.5) binary is also included (`espurna-base.bin`). Like before, the binary included does not have any specific hardware defined. Once flashed using Tuya-Convert you can update to the device-specific version via any of the means that ESPurna provides (OTA, web interface update, update via telnet or MQTT). Please refer to the [ESPurna project page](http://espurna.io) for more info and support.
 
@@ -79,26 +84,24 @@ Requirements:
 * docker is installed
 * docker-compose is installed
 
-Create docker image:
+Preparations:
 * git clone https://github.com/ct-Open-Source/tuya-convert
 * cd tuya-convert
-* docker build -t tuya:latest .
+* if you have already cloned this repo just cd into the directory and execute `git pull`
+* cp .env-template .env 
+* adjust the created .env-file, it contains usage information as comments
 
-Setup docker-compose:
-* copy docker/docker-compose.sample.yml to a new folder you created, the file should be named docker-compose.yml
-* you may adjust this docker-compose.yml, if necessary:
-   * environment-variables may be different, for example network-adapter may be different from wlan0
-   * adjust the volume folder, where you want your backups stored
+Building and running your container:
+* `docker-compose build && docker-compose run --rm tuya`
+* This directly starts into tuya. If you press ctrl+break or exit tuya after flashing, your container is closed and deleted
 
-Run the image:
-* go into the folder you copied docker-compose.yml
-* docker-compose up -d
-* docker-compose exec tuya start
-* tuya-convert now starts within docker
-
-Stop the image:
-* docker-compose exec tuya stop
-* docker-compose down
+Troubleshooting:
+* Q: Where are my logs after flashing? A: The folder can be adjusted in .env with LOCALBACKUPDIR, the path here may be relative or absolute
+* Q: I don't want that my container is deleted after running tuya, I need this for troubleshooting! How do I accomplish this? A: Just remove --rm from `docker-compose run --rm tuya`
+* Q: I want to start the container, but instead of starting tuya immediately I want to get into bash. Is this possible? A: Yes just start the container with `docker-compose run --entrypoint bash tuya`.
+* Q: I want to rebuild my docker-image, even if there are no changes. Is this possible? A: Just start `docker-compose build --no-cache` instead of `docker-compose build`! Don't do this all the time, this is a time consuming process ...
+* Q: I can't connect to my USB, PCI, ... network card. How do I get this working? A: You may have an error in your .env-File. The WLAN-variable should reflect the name of your network interface on your host. Execute ifconfig and look through your interfaces.
+* Q: I can't get an IP-address and or connection on my phone, what's the problem? A: You may look into smarthack-wifi.log (location is set in .env with LOCALBACKUPDIR) or possible stop your firewall (e.g. NixOS seems to have a problem here). It may be a problem with a wrongly set network interface (see previous question)
 
 ## CONTRIBUTING
 

docker-compose.yml

@@ -0,0 +1,12 @@
+version: '3'
+services:
+  tuya:
+    build: .
+    privileged: true
+    network_mode: "host"
+    environment:
+        WLAN: ${WLAN}
+        AP: ${AP}
+        GATEWAY: ${GATEWAY}
+    volumes:
+      - $LOCALBACKUPDIR:/usr/bin/tuya-convert/backups

docker/bin/stop

@@ -1,2 +0,0 @@
-#!/bin/bash
-pkill -SIGINT -f start_flash.sh

docker/bin/tuya-start

@@ -1,3 +1,3 @@
 #!/bin/bash
-cd /usr/bin/tuya-convert
+config-tuya.sh
 ./start_flash.sh

docker/docker-compose.sample.yml

@@ -1,12 +0,0 @@
-version: '3'
-services:
-  tuya:
-    image: tuya:latest
-    privileged: true
-    network_mode: "host"
-    environment:
-      - WLAN=wlan0
-      - AP=vtrust-flash
-      - GATEWAY=10.42.42.1
-    volumes:
-      - ./data/backups:/usr/bin/tuya-convert/backups

install_prereq.sh

@@ -1,10 +1,36 @@
-#!/bin/bash
+#!/usr/bin/env bash
+set -euo pipefail
 
-set -e
+debianInstall() {
+	sudo apt-get update
+	sudo apt-get install -y git iw dnsmasq rfkill hostapd screen curl build-essential python3-pip python3-setuptools python3-wheel python3-dev mosquitto haveged net-tools libssl-dev iproute2 iputils-ping
+	sudo python3 -m pip install --user --upgrade paho-mqtt tornado git+https://github.com/drbild/sslpsk.git pycryptodomex
+}
 
-sudo apt-get update
-sudo apt-get install -y git iw dnsmasq hostapd screen curl build-essential python3-pip python3-setuptools python3-wheel python3-dev mosquitto haveged net-tools libssl-dev
+archInstall() {
+	sudo pacman -S --needed git iw dnsmasq hostapd screen curl python-pip python-wheel python-pycryptodomex python-paho-mqtt python-tornado mosquitto haveged net-tools openssl
+	sudo python -m pip install --user --upgrade git+https://github.com/drbild/sslpsk.git
+}
 
-sudo -H python3 -m pip install --upgrade paho-mqtt tornado git+https://github.com/drbild/sslpsk.git pycryptodomex
+if [[ -e /etc/os-release ]]; then
+	source /etc/os-release
+else
+	echo "/etc/os-release not found! Assuming debian-based system, but this will likely fail!"
+	ID=debian
+fi
+
+if [[ ${ID} == 'debian' ]] || [[ ${ID_LIKE-} == 'debian' ]]; then
+	debianInstall
+elif [[ ${ID} == 'arch' ]] || [[ ${ID_LIKE-} == 'arch' ]]; then
+	archInstall
+else
+	if [[ -n ${ID_LIKE-} ]]; then
+		printID="${ID}/${ID_LIKE}"
+	else
+		printID="${ID}"
+	fi
+	echo "/etc/os-release found but distribution ${printID} is not explicitly supported. Assuming debian-based system, but this will likely fail!"
+	debianInstall
+fi
 
 echo "Ready to start upgrade"

scripts/dr_tuya.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+
+NON_ESP_WARNING="WARNING: it appears this device does not use an ESP82xx and therefore cannot install ESP based firmware"
+
+get_macs_from_wifi_log () {
+	grep AP-STA-CONNECTED smarthack-wifi.log |
+	cut -d ' ' -f3 |
+	sort | uniq
+}
+
+get_ouis_from_wifi_log () {
+	get_macs_from_wifi_log |
+	tr -d : | grep -Po "^.{6}"
+}
+
+psk_02 () {
+	grep -q "ID: 02" smarthack-psk.log
+}
+
+web_non_esp () {
+	grep -q "$NON_ESP_WARNING" smarthack-web.log
+}
+
+udp_non_esp () {
+	grep -q "$NON_ESP_WARNING" smarthack-udp.log
+}
+
+esp_oui () {
+	get_ouis_from_wifi_log | grep -iqf- oui/esp.txt
+}
+
+non_esp_oui () {
+	get_ouis_from_wifi_log | grep -iqf- oui/nonesp.txt
+}
+
+check_for_common_issues () {
+	if psk_02; then
+		echo "Your device's firmware is too new."
+		echo "Tuya patched the PSK vulnerability that we use to establish a connection."
+		echo "You might still be able to flash this device over serial."
+		echo "For more information and to follow progress on solving this issue see:"
+		echo "https://github.com/ct-Open-Source/tuya-convert/wiki/Collaboration-document-for-PSK-Identity-02"
+		exit 1
+	fi
+	if web_non_esp || udp_non_esp || non_esp_oui; then
+		echo "Your device does not use an ESP82xx."
+		echo "This means you cannot flash custom ESP firmware even over serial."
+		exit 1
+	fi
+	if esp_oui; then
+		echo "An ESP82xx based device connected according to your wifi log."
+		echo "If this is the device you are trying to flash, another issue may be preventing it from flashing."
+		echo "Otherwise, it could be that the device does not use an ESP82xx or it did not connect."
+	else
+		echo "No ESP82xx based devices connected according to your wifi log."
+		echo "Here is a list of all the MAC addresses that connected:"
+		get_macs_from_wifi_log
+		echo
+		echo "If you see your IoT device in this list, it is not an ESP82xx based device."
+		echo "Otherwise, another issue may be preventing it from connecting."
+	fi
+	echo "For additional information, check the *.log files inside the scripts folder."
+	echo "Please include these logs when opening a new issue on our GitHub issue tracker."
+}
+
+check_for_common_issues
+

scripts/mosquitto.conf

@@ -0,0 +1,2 @@
+allow_anonymous true
+listener 1883

scripts/old_screen_with_log.sh

@@ -0,0 +1,21 @@
+# Run screen with a custom .screenrc that defines the name of "logfile"
+
+screen_minor=${1}
+screen_logfile_name=${2}
+screen_other_options=${@:3}
+
+if [ "$screen_minor" -gt 5 ]; then
+	echo "Info: you have the modern enough version" \
+             "to use the \"-Logfile\" flag of \"screen\""
+elif [ "$screen_minor" -eq 5 ]; then
+	screen_with_log="sudo screen -L"
+else
+	screen_with_log="sudo screen -L -t"
+fi
+
+echo "logfile ${screen_logfile_name}" > ${screen_logfile_name}.screenrc
+
+${screen_with_log} ${screen_logfile_name} \
+    -c ${screen_logfile_name}.screenrc ${screen_other_options}
+
+rm ${screen_logfile_name}.screenrc

scripts/oui/esp.txt

@@ -0,0 +1,70 @@
+# "Espressif Inc." OUIs
+# updated Jan 26 2021
+083AF2
+0CDC7E
+10521C
+18FE34
+240AC4
+2462AB
+246F28
+24A160
+24B2DE
+2C3AE8
+2CF432
+30AEA4
+3C6105
+3C71BF
+40F520
+483FDA
+4C11AE
+4C7525
+500291
+545AA6
+5CCF7F
+600194
+6055F9
+68C63A
+70039F
+7C9EBD
+7CDFA1
+807D3A
+840D8E
+84CCA8
+84F3EB
+8CAAB5
+8CCE4E
+9097D5
+943CC6
+94B97E
+98CDAC
+98F4AB
+9C9C1F
+A020A6
+A0764E
+A47B9D
+A4CF12
+A4E57C
+A8032A
+AC67B2
+ACD074
+B4E62D
+B8F009
+BCDDC2
+BCFF4D
+C44F33
+C45BBE
+C4DD57
+C82B96
+CC50E3
+D8A01D
+D8BFC0
+D8F15B
+DC4F22
+E09806
+E0E2E6
+E868E7
+E8DB84
+ECFABC
+F008D1
+F4CFA2
+FCF5C4

scripts/oui/nonesp.txt

@@ -0,0 +1,18 @@
+# Known unsupported OUIs including
+# "Tuya (HK) Limietd"
+# "Tuya Smart Inc."
+# "Beijing Winner Microelectronics Ltd"
+# updated Jan 26 2021
+000D8A
+000F89
+10D561
+1869D8
+286DCD
+68572D
+708976
+807484
+84E342
+D4A651
+DC446D
+E078A3
+CC8CBF

scripts/psk-frontend.py

@@ -69,7 +69,7 @@ class PskFrontend():
 			self.sessions.append((ssl_sock, s2))
 		except ssl.SSLError as e:
 			print("could not establish sslpsk socket:", e)
-			if "NO_SHARED_CIPHER" in e.reason or "WRONG_VERSION_NUMBER" in e.reason or "WRONG_SSL_VERSION" in e.reason:
+			if e and ("NO_SHARED_CIPHER" in e.reason or "WRONG_VERSION_NUMBER" in e.reason or "WRONG_SSL_VERSION" in e.reason):
 				print("don't panic this is probably just your phone!")
 		except Exception as e:
 			print(e)

scripts/setup_ap.sh

@@ -26,7 +26,13 @@ setup () {
 
 	if test -d /etc/NetworkManager; then
 		echo "Stopping NetworkManager..."
-		sudo service network-manager stop
+		if ! sudo systemctl stop network-manager 2>/dev/null
+		then
+			if ! sudo systemctl stop NetworkManager 2>/dev/null
+			then
+				echo "** Failed to stop NetworkManager, AP may not work! **"
+			fi
+		fi
 	fi
 
 	echo "Configuring AP interface..."
@@ -42,6 +48,7 @@ setup () {
 		--interface=$WLAN \
 		--bind-interfaces \
 		--listen-address=$GATEWAY \
+		--except-interface=lo \
 		--dhcp-range=10.42.42.10,10.42.42.40,12h \
 		--address=/#/$GATEWAY
 
@@ -61,11 +68,16 @@ cleanup () {
 
 	if test -d /etc/NetworkManager; then
 		echo "Restarting NetworkManager..."
-		sudo service network-manager restart
+		if ! sudo systemctl restart network-manager 2>/dev/null
+		then
+			if ! sudo systemctl restart NetworkManager 2>/dev/null
+			then
+				echo "** Failed to restart NetworkManager: network may not be functional! **"
+			fi
+		fi
 	fi
 }
 
 version_check
 trap cleanup EXIT
 setup
-

scripts/setup_checks.sh

@@ -67,7 +67,8 @@ check_port () {
 	port="$2"
 	reason="$3"
 	echo -n "Checking ${protocol^^} port $port... "
-	process_pid=$(sudo ss -lnp -A "$protocol" "sport = :$port" | grep -Po "(?<=pid=)(\d+)" | head -n1)
+	ss_output=$(sudo ss -lnp -A "$protocol" "sport = :$port")
+	process_pid=$(echo "$ss_output" | grep -Po "(?<=pid=)(\d+)" | head -n1)
 	if [ -n "$process_pid" ]; then
 		process_name=$(ps -p "$process_pid" -o comm=)
 		echo "Occupied by $process_name with PID $process_pid."
@@ -98,6 +99,12 @@ check_port () {
 			sleep 1
 		fi
 	else
+		if [ $(echo "$ss_output" | wc -l) -gt 1 ]; then
+			echo "Occupied by unknown process."
+			echo "Port $port is needed to $reason"
+			echo "Aborting due to occupied port"
+			exit 1
+		fi
 		echo "Available."
 	fi
 }

start_flash.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 bold=$(tput bold)
 normal=$(tput sgr0)
 . ./config.txt
@@ -10,10 +10,8 @@ setup () {
 	screen_minor=$(screen --version | cut -d . -f 2)
 	if [ "$screen_minor" -gt 5 ]; then
 		screen_with_log="sudo screen -L -Logfile"
-	elif [ "$screen_minor" -eq 5 ]; then
-		screen_with_log="sudo screen -L"
 	else
-		screen_with_log="sudo screen -L -t"
+		screen_with_log="./old_screen_with_log.sh ${screen_minor}"
 	fi
 	echo "======================================================"
 	echo -n "  Starting AP in a screen"
@@ -26,7 +24,7 @@ setup () {
 	echo "  Starting web server in a screen"
 	$screen_with_log smarthack-web.log -S smarthack-web -m -d ./fake-registration-server.py
 	echo "  Starting Mosquitto in a screen"
-	$screen_with_log smarthack-mqtt.log -S smarthack-mqtt -m -d mosquitto -v
+	$screen_with_log smarthack-mqtt.log -S smarthack-mqtt -m -d mosquitto -v -c $PWD/mosquitto.conf
 	echo "  Starting PSK frontend in a screen"
 	$screen_with_log smarthack-psk.log -S smarthack-psk -m -d ./psk-frontend.py -v
 	echo "  Starting Tuya Discovery in a screen"
@@ -75,13 +73,16 @@ while true; do
 	# The intermediate firmware will request 10.42.42.42
 	# Do NOT change this address!!!
 	# It will NOT make it install and will break this script
-	while ! ping -c 1 -W 1 -n 10.42.42.42 &> /dev/null; do
+	while ! ping -c 1 -W 1 -n 10.42.42.42 -I 10.42.42.1 &> /dev/null; do
 		printf .
 		if (( --i == 0 )); then
 			echo
-			echo "Device did not appear with the intermediate firmware"
-			echo "Check the *.log files in the scripts folder"
+			echo "Timed out while waiting for the device to (re)connect"
 			pkill -f smartconfig/main.py && echo "Stopping smart config"
+			echo "======================================================"
+			echo "Attempting to diagnose the issue..."
+			./dr_tuya.sh
+			echo "======================================================"
 			read -p "Do you want to try flashing another device? [y/N] " -n 1 -r
 			echo
 			[[ "$REPLY" =~ ^[Yy]$ ]] || break 2
@@ -118,8 +119,8 @@ while true; do
 	echo "Ready to flash third party firmware!"
 	echo
 	echo "For your convenience, the following firmware images are already included in this repository:"
-	echo "  Tasmota v8.1.0.2 (wifiman)"
-	echo "  ESPurna 1.13.5 (base)"
+	echo "  Tasmota v9.2.0 (wifiman)"
+	echo "  ESPurna 1.5 (base)"
 	echo
 	echo "You can also provide your own image by placing it in the /files directory"
 	echo "Please ensure the firmware fits the device and includes the bootloader"
@@ -133,4 +134,3 @@ while true; do
 	echo
 	[[ "$REPLY" =~ ^[Yy]$ ]] || break
 done
-