개발환경 구축 이슈 메모

Environment

java 11 (openjdk11)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29

# [ java 11 install ]

# 1. 우분투 버전 확인
lsb_release -a
# mine > Description:    Ubuntu 22.04.2 LTS

# 2. 22.04 default 인 java 설치 - jre & jdk
sudo apt install default-jre
java -version

sudo apt install default-jdk
javac -version

# 3. /etc/profile 에 아래 JAVA_HOME 추가
cd /usr/lib/jvm/
sudo ln -s java-11-openjdk-amd64 jdk #jdk 링크 생성
sudo vi /etc/profile

# in /etc/profile, add java path
############ JDK 11 PATH #############
JAVA_HOME=/usr/lib/jvm/jdk
CLASSPATH=.:$JAVA_HOME/lib/tools.jar
PATH=$PATH:$HOME/bin:$JAVA_HOME/bin
export JAVA_H0ME CLASSPATH
export PATH

source /etc/profile

javac -version # 확인

python 3.9(anaconda3)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17

# [ python 3.9 install ]

# 1. wget 으로 ananconda 3 2021.11 배포판 다운 후 실행
wget https://repo.anaconda.com/archive/Anaconda3-2021.11-Linux-x86_64.sh
sudo bash Anaconda3-2021.11-Linux-x86_64.sh 
    # 설치 중 라이센스는 yes로 승낙
    # 설치폴더는 /root/anaconda3 말고 /usr/lib/python/anaconda3 로 설정

# 2. /etc/profile 재설정
sudo vi /etc/profile
######### PYTHON PATH ############
PYTH0N_H0ME=/usr/lib/python/anaconda3
export PATH=$PATH:$PYTH0N_H0ME/bin

source /etc/profile

python --version

database (MySQL, MongoDB)

1
2
3
4
5
6

# MySQL
sudo apt-get update

sudo apt-get install mysql-server -y

sudo mysql_secure_installation

1
2
3
4
5
6
7
8
9
10
11
12
13

CREATE DATABASE IF NOT EXISTS etlmysql;

CREATE USER '{username}'@'localhost' IDENTIFIED BY '{password}';

FLUSH PRIVILEGES;

GRANT ALL PRIVILEGES ON etlmysql.* to '{username}'@'localhost';

FLUSH PRIVILEGES;

SHOW GRANTS FOR '{username}'@'localhost';


1
2

sudo dpkg -i mysql-connector-java_8.0.30-1ubuntu22.04_all.deb
ls -l /usr/share/java/mysql-connector-java-8.0.30.jar

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31

# MongoDB
su
sudo apt-get install gnupg

# import the MongoDB key from the official website
curl -fsSL https://pgp.mongodb.com/server-6.0.asc | \
sudo gpg -o /usr/share/keyrings/mongodb-server-6.0.gpg \
--dearmor

# mongodb-org-6.0.list” in “source.list.d” directory to import the MongoDB repository
echo "deb [ arch=amd64,arm64 signed-by=/usr/share/keyrings/mongodb-server-6.0.gpg ] https://repo.mongodb.org/apt/ubuntu jammy/mongodb-org/6.0 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-6.0.list

sudo apt-get update
sudo apt-get install -y mongodb-org

sudo systemctl daemon-reload
sudo systemctl start mongod
sudo systemctl status mongod


# Mongodb Compass install
wget https://downloads.mongodb.com/compass/mongodb-compass_1.35.0_amd64.deb

sudo dpkg -i mongodb-compass_1.35.0_amd64.deb

sudo apt-get install libgconf-2-4

sudo apt --fix-broken install

# start Mongodb Compass
mongodb-compass

HADOOP

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174

# open-ssh 설치 && ssh 접속
sudo apt install openssh-server openssh-client -y

ssh-keygen -t rsa
cd /home/lsyeup1206/.ssh/
cp id_rsa.pub authorized_keys
# ssh localhost

# hadoop 설치
wget https://downloads.apache.org/hadoop/common/hadoop-3.3.2/hadoop-3.3.2.tar.gz
tar zxvf hadoop-3.3.2.tar.gz
ln -s hadoop-3.3.2 hadoop

# hadoop 환경변수 설정
vi .bashrc

# ######## HADOOP-3.3.2 PATH ############ 
HAD00P_H0ME=/home/lsyeup1206/hadoop PATH=$PATH:$HAD00P_H0ME/bin:$HAD00P_H0ME/sbin 
export PATH
export HADOOP_MAPRED_HOME=$HADOOP_HOME
export HADOOP_COMMON_HOME=$HADOOP_HOME
export HADOOP_HDFS_HOME=$HADOOP_HOME
export HAD00P_YARN_H0ME=$HAD00P_H0ME
export HAD00P_C0NF_DIR=$HAD00P_H0ME/etc/hadoop
export HDFS_NAMENODE_USER=lsyeup1206
export HDFS_DATANODE_USER=lsyeup1206
export HDFS_SECONDARYNAMENODE_USER=lsyeup1206 
export YARN_RESOURCEMANAGER_USER=lsyeup1206 
export YARN_NODEMANAGER_USER=lsyeup1206

source .bashrc


# hadoop 설정파일 세팅
cd /home/lsyeup1206/hadoop/etc/hadoop/

vi hadoop-env.sh
export JAVA_HOME=/usr/lib/jvm/jdk

vi yarn-env.sh
export JAVA_HOME=/usr/lib/jvm/jdk

vi core-site.xml
<configuration>
        <property>
                <name>fs.defaultFS</name>
                <value>hdfs://localhost:9000</value>
        </property>
        <property>
                <name>hadoop.tmp.dir</name>
                <value>/home/lsyeup1206/tmp</value>
        </property>
</configuration>

vi hdfs-site.xml
<configuration>
        <!-- 기본 설정-->
        <property>
                <name>dfs.replication</name>
                <value>1</value>
        </property>
        <property>
                <name>dfs.name.dir</name>
                <value>file:///home/lsyeup1206/hadoop/hdfs/namenode</value>
        </property>
        <property>
                <name>dfs.data.dir</name>
                <value>file:///home/lsyeup1206/hadoop/hdfs/datanode</value>
        </property>

        <!-- datanode가 1개인 상황에서 datanode의 추가없이 기존 datanode 에서 작업-->
        <property>
                <name>dfs.client.block.write.replace-datanode-on-failure.enable</name>
                <value>true</value>
        </property>
        <property>
                <name>dfs.client.block.write.replace-datanode-on-failure.policy</name>
                <value>NEVER</value>
        </property>
        <property>
                <name>dfs.client.block.write.replace-datanode-on-failure.best-effort</name>
                <value>true</value>
        </property>
</configuration>

vi mapred-site.xml
<configuration>
        <property>
                <name>mapreduce.framework.name</name>
                <value>yarn</value>
        </property>
</configuration>

vi yarn-site.xml
<configuration>
        <!-- Site specific YARN configuration properties -->
        <property>
                <name>yarn.nodemanager.aux-services</name>
                <value>mapreduce_shuffle</value>
        </property>
</configuration>

# namenode, datanode 담당할 폴더 생성
mkdir -p $HADOOP_HOME/hdfs/namenode
mkdir -p $HADOOP_HOME/hdfs/datanode

# namenode format
hdfs namenode -format

# hadoop3 실행 확인
start-dfs.sh
start-yarn.sh
jps

# Ubuntu 부팅 시 자동으로 하둡 서비스가 실행할 수 있도록 !

vi /home/lsyeup1206/hadoop/sbin/hadoop-service.sh

#! /bin/bash
# start() {
#     export HADOOP_HOME=/home/lsyeup1206/hadoop
#     export PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin

#     start-dfs.sh
#     start-yarn.sh
# }

# stop() {
#     export HADOOP_HOME=/home/lsyeup1206/hadoop
#     export PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin

#     stop-dfs.sh
#     stop-yarn.sh
# }

# case $1 in
#     start|stop "$1" ;;
# esac

# exit

# 실행 권한부여
chmod +x /home/lsyeup1206/hadoop/sbin/hadoop-service.sh

# 우분투 systemctl에 hadoop-service.sh 를 등록한다.
sudo vi /usr/lib/systemd/system/hadoop.service

[Unit]
Description=Hadoop 3.3.2 Service

[Service]
Type=oneshot

User=lsyeup1206
Group=lsyeup1206

ExecStart=/home/lsyeup1206/hadoop/sbin/hadoop-service.sh start
ExecStop=/home/lsyeup1206/hadoop/sbin/hadoop-service.sh stop

RemainAfterExit=yes
#Restart=always

[Install]
WantedBy=multi-user.target

sudo systemctl enable hadoop
sudo systemctl start hadoop
sudo systemctl status hadoop

# hdfs jps명령어 입력 시 전부 안올라오는 경우
# 아래 경로에 로그파일들 tail걸어 놓고, bindException 뜨는 포트들 킬해준다.
/home/lsyeup1206/hadoop/logs
lsof -i -P -n | grep LISTEN

추가로 해줘야 할 설정

1
2
3
4
5
6

# 나는 WSL2에 우분투로 하둡 설치하고 사용중이므로, 추가되는 포트들에 대해 아래 설정들 추가해준다.

1. 방화벽 포트포워딩 
2. 인바운드 규칙 허용
3. localhost : wsl2 간 포트포워딩 (관리자권한: wsl2_portforward_script.ps1)