te')); return $arr; } /* 遍历用户所有主题 * @param $uid 用户ID * @param int $page 页数 * @param int $pagesize 每页记录条数 * @param bool $desc 排序方式 TRUE降序 FALSE升序 * @param string $key 返回的数组用那一列的值作为 key * @param array $col 查询哪些列 */ function thread_tid_find_by_uid($uid, $page = 1, $pagesize = 1000, $desc = TRUE, $key = 'tid', $col = array()) { if (empty($uid)) return array(); $orderby = TRUE == $desc ? -1 : 1; $arr = thread_tid__find($cond = array('uid' => $uid), array('tid' => $orderby), $page, $pagesize, $key, $col); return $arr; } // 遍历栏目下tid 支持数组 $fid = array(1,2,3) function thread_tid_find_by_fid($fid, $page = 1, $pagesize = 1000, $desc = TRUE) { if (empty($fid)) return array(); $orderby = TRUE == $desc ? -1 : 1; $arr = thread_tid__find($cond = array('fid' => $fid), array('tid' => $orderby), $page, $pagesize, 'tid', array('tid', 'verify_date')); return $arr; } function thread_tid_delete($tid) { if (empty($tid)) return FALSE; $r = thread_tid__delete(array('tid' => $tid)); return $r; } function thread_tid_count() { $n = thread_tid__count(); return $n; } // 统计用户主题数 大数量下严谨使用非主键统计 function thread_uid_count($uid) { $n = thread_tid__count(array('uid' => $uid)); return $n; } // 统计栏目主题数 大数量下严谨使用非主键统计 function thread_fid_count($fid) { $n = thread_tid__count(array('fid' => $fid)); return $n; } ?>amazon web services - Connection from Docker Container to AWS RDS PostgreSQL Failed - Stack Overflow

科技改变生活-雨落星辰 - 所有的伟大,都源于一个勇敢的开始

    最新消息：雨落星辰是一个专注网站SEO优化、网站SEO诊断、搜索引擎研究、网络营销推广、网站策划运营及站长类的自媒体原创博客
    你的位置： 首页>programmer>amazon web services - Connection from Docker Container to AWS RDS PostgreSQL Failed - Stack Overflow

    amazon web services - Connection from Docker Container to AWS RDS PostgreSQL Failed - Stack Overflow

    programmeradmin5浏览0评论

    I am trying to connect to a PostgreSQL database on AWS RDS from a spark master container running on docker.

    I installed both the AWS CLI and the AWS session manager plugin inside the spark master container to be able to connect from the container itself to the DB.

    Additionally,the spark master container is responsible for submitting a spark job that will make use of the port forwarding ( aws ssm start-session ) we did in the same container and connect to the DB.

    Now when I run AWS configure and aws ssm start-session commands inside the container, I get

    Starting session with session id xxx.
Port xxxx opened for session ID XXX
Waiting for connections...

    When I run the spark-submit right after on the same container, I get Connection accepted for session xxxx from the aws ssm terminal, then right after, the spark job stops with an error:

    .postgresql.util.PSQLException: Connection to localhost:5555 refused. Check that the hostname and port are correct and that the postmaster is accepting TCP/IP connections.

    So it looks like connection is being accepted for a really short period of time, then refused? two processes are reporting two completely different results.

    I am trying to connect to a PostgreSQL database on AWS RDS from a spark master container running on docker.

    I installed both the AWS CLI and the AWS session manager plugin inside the spark master container to be able to connect from the container itself to the DB.

    Additionally,the spark master container is responsible for submitting a spark job that will make use of the port forwarding ( aws ssm start-session ) we did in the same container and connect to the DB.

    Now when I run AWS configure and aws ssm start-session commands inside the container, I get

    Starting session with session id xxx.
Port xxxx opened for session ID XXX
Waiting for connections...

    When I run the spark-submit right after on the same container, I get Connection accepted for session xxxx from the aws ssm terminal, then right after, the spark job stops with an error:

    .postgresql.util.PSQLException: Connection to localhost:5555 refused. Check that the hostname and port are correct and that the postmaster is accepting TCP/IP connections.

    So it looks like connection is being accepted for a really short period of time, then refused? two processes are reporting two completely different results.

    • postgresql
    • amazon-web-services
    • docker
    • apache-spark
    • docker-compose
    Share Improve this question asked 2 days ago EyaEya 135 bronze badges 5
    • AWS CLI and AWS session manager plugin are for creating secure tunnels between the container and the DB. The actual connection is established inside the spark job itself @MarkB I wasn't precise enough in the post, my bad. – Eya Commented 2 days ago
    • My first guess would be that the aws ssm start-session process is being stopped before the next process (spark) is started. Can you show your entire Docker startup script so we can see how you are running both of these in the docker container? – Mark B Commented 2 days ago
    • @MarkB here is the detailed setup link, and about the ssm process, I don't think it is being stopped, because the corresponding terminal indicates a still running process after it accepts the connection and after I get connection error from the spark job. – Eya Commented 2 days ago
    • You couldn't edit this question to include those details? You had to create an entirely new question? – Mark B Commented 2 days ago
    • The link I provided corresponds to a post I made before this one about the same problem, I thought maybe it's too long of a post to actually be read and answered so I tried to be more concise in the current one.@MarkB – Eya Commented 2 days ago
    Add a comment  | 

    1 Answer 1

    Reset to default 0

    Ok, I resolved the issue and I am leaving what worked for me in case someone faces the same problem in the future.

    What I did wrong is running the aws ssm start-session process only in the spark master container, the connection accepted message was the master's connection being accepted, but spark works with lazy transformations, so when the master actually finds actions to be executed ( the sales.show(10) in my case ), the workers are the ones who will do the actual work, so we have spark worker containers that need a piece of data not read yet ( because lazy transformations ) to manipulate, yet they are not connected to the DB. So the connection accepted is the master's connection being accepted, and the connection error was the spark workers reporting back to the master, hence the contradiction.

    The solution is to run aws ssm start-session in both master and worker containers. And each worker container will connect separately to the DB whenever assigned with a piece of work involving data for the first time.

    与本文相关的文章

    发布评论

    评论列表(0)

    1. 暂无评论